rubber 1.15.0 → 2.0.0.pre1

data/templates/graylog/config/rubber/role/common/graylog-rsyslog.conf

@@ -0,0 +1,32 @@
+<%
+  @path = "/etc/rsyslog.d/1-graylog.conf"
+  @post = "/usr/sbin/service rsyslog restart"
+
+  graylog_server = rubber_instances.for_role("graylog_server").first
+  is_graylog_instance = rubber_instances[rubber_env.host].role_names.include?('graylog_server')
+%>
+
+<% if graylog_server %>
+
+# start forwarding rule - we want  logs to buffer up if
+# remote log collector not available
+$ActionQueueType LinkedList
+$ActionQueueFileName graylog-buffer
+$ActionQueueMaxDiskSpace 1g
+$ActionResumeRetryCount -1
+$ActionQueueSaveOnShutdown on
+
+# the graylog instance's rsyslog collects all syslog in local syslog daemon and
+# then forwards it on to local graylog daemon
+<% if is_graylog_instance %>
+
+*.* @<%= graylog_server.full_name %>:<%= rubber_env.graylog_server_syslog_port %>
+
+<% else %>
+
+*.* @<%= graylog_server.full_name %>:514
+
+<% end %>
+# end forwarding rule 1
+
+<% end %>

data/templates/graylog/config/rubber/role/graylog_server/graylog2.conf

@@ -0,0 +1,67 @@
+<%
+  @path = "/etc/graylog2.conf"
+%>
+
+# On which port (UDP) should we listen for Syslog messages? (Standard: 514)
+syslog_listen_port = <%= rubber_env.graylog_server_syslog_port %>
+syslog_protocol = udp
+
+# ElasticSearch URL (default: http://localhost:9200/)
+elasticsearch_url = http://<%= rubber_instances.for_role('graylog_elasticsearch').first.full_name %>:9200/
+elasticsearch_index_name = <%= rubber_env.graylog_elasticsearch_index %>
+
+# Always try a reverse DNS lookup instead of parsing hostname from syslog message?
+force_syslog_rdns = false
+
+# MongoDB Configuration
+mongodb_useauth = false
+mongodb_user = grayloguser
+mongodb_password = 123
+mongodb_host = <%= rubber_instances.for_role('graylog_mongodb').first.full_name %>
+#mongodb_replica_set = localhost:27017,localhost:27018,localhost:27019
+mongodb_database = <%= rubber_env.graylog_mongo_database %>
+mongodb_port = 27017
+
+# Graylog2 uses an internal message queue that holds all received messages until they are indexed. The mq_batch_size parameter defines how many messages are sent
+# to ElasticSearch at once (using a _bulk update: http://www.elasticsearch.org/guide/reference/api/bulk.html). The mq_poll_freq parameter controls in which
+# interval (in seconds) the message batch is sent. Example: If you leave the standard values (mq_batch_size = 4000, mq_poll_freq = 1), Graylog2 will index 4000 messages
+# every second. If you have spikes with more than 4000 messages per second, the queue will start growing until you get under 4000 messages/second again. The queue is
+# FIFO and can grow until you run out of RAM. Note that the queue *only* resists in RAM, so if you set the mq_poll_freq to a high value, you may lose a lot of not yet
+# indexed messages when the server crashes. Run the server in debug mode (java -jar graylog2-server.jar --debug) with a |grep '^INFO' to see debug information about
+# the queue and it's size. (INFO : org.graylog2.periodical.BulkIndexerThread - About to index max 4000 messages. You have a total of 103 messages in the queue. [freq:1s])
+# You can also monitor the queue size in your graylog2-web-interface.
+mq_batch_size = 4000
+mq_poll_freq = 1
+
+# You can set a maximum size of the message queue. If this size is reached, all new messages will be rejected until messages are removed/indexed from the queue.
+# 0 = unlimited queue size (default)
+mq_max_size = 0
+
+# Raise this according to the maximum connections your MongoDB server can handle if you encounter MongoDB connection problems.
+mongodb_max_connections = 100
+
+# Number of threads allowed to be blocked by MongoDB connections multiplier. Default: 5
+# If mongodb_max_connections is 100, and mongodb_threads_allowed_to_block_multiplier is 5, then 500 threads can block. More than that and an exception will be thrown.
+# http://api.mongodb.org/java/current/com/mongodb/MongoOptions.html#threadsAllowedToBlockForConnectionMultiplier
+mongodb_threads_allowed_to_block_multiplier = 5
+
+# Graylog Extended Log Format (GELF)
+use_gelf = true
+gelf_listen_address = 0.0.0.0
+gelf_listen_port = <%= rubber_env.graylog_server_port %>
+
+# Drools Rule File (Use to rewrite incoming log messages)
+# rules_file = /etc/graylog2.d/rules/graylog2.drl
+
+# AMQP
+amqp_enabled = false
+amqp_subscribed_queues = somequeue1:gelf,somequeue2:gelf,somequeue3:syslog
+amqp_host = localhost
+amqp_port = 5672
+amqp_username = guest
+amqp_password = guest
+amqp_virtualhost = /
+
+# Forwarders
+# Timeout in seconds for each connection and read of Logg.ly API when forwarding messages. Default: 3
+forwarder_loggly_timeout = 3

data/templates/graylog/config/rubber/role/graylog_server/graylog_server-upstart.conf

@@ -0,0 +1,22 @@
+<%
+  @path = "/etc/init/graylog-server.conf"
+  @backup = false
+%>
+description "graylog server"
+
+start on (started mongodb and started elasticsearch)
+stop on runlevel [016]
+
+script
+  cd <%= rubber_env.graylog_server_dir %>
+  ulimit -n 65536
+  exec java -jar graylog2-server.jar > graylog.log 2>&1
+end script
+
+post-start script
+  status graylog-server | head -n1 | awk '{print $NF}' > <%= rubber_env.graylog_server_pid_file %>
+end script
+
+post-stop script
+  rm -f <%= rubber_env.graylog_server_pid_file %>
+end script

data/templates/graylog/config/rubber/role/graylog_web/crontab

@@ -0,0 +1,9 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ['# start-graylog-web-crontab', '# end-graylog-web-crontab']
+%>
+
+# use sudo/bash to unset bundle env vars since we are running a different
+# rails app and redirect output to null to prevent bash-login from complaining about tty
+*/15 * * * * sudo -u root bash -l -c "<%= Rubber.root %>/script/rubber cron -r <%= rubber_env.graylog_web_dir %> bundle exec rake subscriptions:send" &> /dev/null

data/templates/graylog/config/rubber/role/graylog_web/email.yml

@@ -0,0 +1,16 @@
+<%
+  @path = "#{rubber_env.graylog_web_dir}/config/email.yml"
+%>
+
+# Config for pony email gem
+
+production:
+  via: sendmail
+#  via: smtp # via: sendmail
+#  host: smtp.example.org
+#  enable_starttls_auto: true
+#  port: 25
+#  auth: plain # plain, login, cram_md5 - Comment out or remove to use no auth
+#  user: somebody
+#  password: yourpass
+#  domain: example.org # the HELO domain provided by the client to the server 

data/templates/graylog/config/rubber/role/graylog_web/general.yml

@@ -0,0 +1,27 @@
+<%
+  @path = "#{rubber_env.graylog_web_dir}/config/general.yml"
+  graylog_server = rubber_instances.for_role("graylog_server").first
+  webtools_server = rubber_instances.for_role("web_tools").first
+%>
+
+general:
+  external_hostname: "<%= webtools_server.full_name %>:<%= rubber_env.web_tools_port %>" # Used for example to generate permalinks. Don't add 'http://' or trailing slashes.
+  date_format: "%d.%m.%Y - %H:%M:%S" # http://ruby-doc.org/core/classes/Time.html#M000298 (strftime syntax)
+  allow_deleting: false # Allowing deleting of messages negatively impacts performance
+  allow_version_check: true # Enables manual (/versioncheck/index) and automatic (every 30min from overview page) version checking against graylog2.org via HTTP. 
+  # custom_cookie_name: graylog2_staging1 # Set an own cookie name - Useful for multiple deployments on same host like example.org/staging1/graylog2, example.org/staging2/graylog2
+
+# Settings for stream subscription emails. 
+subscriptions:
+  from: graylog2subscriptions@<%= rubber_env.domain %>
+  subject: "[graylog2] Subscription"
+
+# Settings for stream alarm emails.
+streamalarms:
+  from: graylog2alarms@<%= rubber_env.domain %>
+  subject: "[graylog2] Stream alarm!"
+
+hoptoad:
+  enabled: false
+  ssl: false
+  api_key: 123

data/templates/graylog/config/rubber/role/graylog_web/graylog_web-upstart.conf

@@ -0,0 +1,21 @@
+<%
+  @path = "/etc/init/graylog-web.conf"
+  @backup = false
+%>
+description "graylog web"
+
+start on started graylog-server
+stop on runlevel [016]
+
+script
+  cd <%= rubber_env.graylog_web_dir %>
+  exec bash -l -c "passenger start -e production --user <%= rubber_env.app_user %> -p <%= rubber_env.graylog_web_port %> --log-file <%= rubber_env.graylog_web_dir %>/log/server.log"
+end script
+
+post-start script
+  status graylog-web | head -n1 | awk '{print $NF}' > <%= rubber_env.graylog_web_pid_file %>
+end script
+
+post-stop script
+  rm -f <%= rubber_env.graylog_web_pid_file %>
+end script

data/templates/graylog/config/rubber/role/graylog_web/mongoid.yml

@@ -0,0 +1,28 @@
+<%
+  @path = "#{rubber_env.graylog_web_dir}/config/mongoid.yml"
+  graylog_server = rubber_instances.for_role("graylog_server").first
+%>
+defaults: &defaults
+  host: localhost
+  allow_dynamic_fields: true
+  # slaves:
+  #   - host: slave1.local
+  #     port: 27018
+  #   - host: slave2.local
+  #     port: 27019
+
+development:
+  <<: *defaults
+  database: graylog2_development
+
+test:
+  <<: *defaults
+  database: graylog2_test
+
+# set these environment variables on your prod server
+production:
+  host: <%= graylog_server.full_name %>
+  port: 27017
+  username:
+  password:
+  database: <%= rubber_env.graylog_mongo_database %>

data/templates/graylog/config/rubber/rubber-graylog.yml

@@ -0,0 +1,42 @@
+graylog_server_version: "0.9.6"
+graylog_server_pkg_url: "https://github.com/downloads/Graylog2/graylog2-server/graylog2-server-#{graylog_server_version}.tar.gz"
+graylog_server_prefix: "/usr/local"
+graylog_server_dir: "#{graylog_server_prefix}/graylog2-server-#{graylog_server_version}"
+graylog_server_pid_file: "/var/run/graylog-server.pid"
+graylog_server_port: 12201
+graylog_server_syslog_port: 12514
+
+graylog_web_version: "0.9.6"
+graylog_web_pkg_url: "https://github.com/downloads/Graylog2/graylog2-web-interface/graylog2-web-interface-#{graylog_web_version}.tar.gz"
+graylog_web_prefix: "/usr/local"
+graylog_web_dir: "#{graylog_web_prefix}/graylog2-web-interface-#{graylog_web_version}"
+graylog_web_pid_file: "/var/run/graylog-web.pid"
+graylog_web_port: 3005
+
+graylog_web_username: admin
+graylog_web_password: admin1
+graylog_web_email: "#{admin_email}"
+
+graylog_mongo_database: graylog2
+graylog_elasticsearch_index: graylog2
+
+web_tools_proxies:
+  graylog:
+    role: graylog_web
+    port: "#{graylog_web_port}"
+    path: /
+
+role_dependencies:
+  web_tools: [graylog_web, graylog_server]
+  graylog_server: [graylog_mongodb, graylog_elasticsearch]
+  graylog_mongodb: [mongodb]
+  graylog_elasticsearch: [elasticsearch]
+
+roles:
+  graylog_server:
+    packages: [openjdk-6-jdk, mongodb-10gen]
+  graylog_web:
+    packages: [libcurl4-openssl-dev]
+    gems: [[passenger, "#{passenger_version || '3.0.7'}"]]
+
+gems: [gelf, graylog2_exceptions, graylog2-resque]

data/templates/graylog/templates.rb

@@ -0,0 +1,5 @@
+if Rubber::Util::is_bundler?
+  append_to_file "Gemfile", "gem 'gelf'\n"
+  append_to_file "Gemfile", "gem 'graylog2_exceptions', :git => 'git://github.com/wr0ngway/graylog2_exceptions.git'\n"
+  append_to_file "Gemfile", "gem 'graylog2-resque'\n"
+end

data/templates/graylog/templates.yml

@@ -0,0 +1,5 @@
+description: The graylog log aggregation and search module
+dependent_templates:
+  - mongodb
+  - elasticsearch
+  

data/{lib/generators/vulcanize/templates → templates}/haproxy/config/rubber/deploy-haproxy.rb

@@ -9,7 +9,7 @@ namespace :rubber do
     # rubber auto-roles don't get defined till after all tasks are defined
     on :load do
       rubber.serial_task self, :serial_restart, :roles => :haproxy do
-        rsudo "service haproxy restart"
+        rsudo "service haproxy stop; service haproxy start"
       end
       rubber.serial_task self, :serial_reload, :roles => :haproxy do
         rsudo "if ! ps ax | grep -v grep | grep -c haproxy &> /dev/null; then service haproxy start; else service haproxy reload; fi"
@@ -22,7 +22,7 @@ namespace :rubber do
     
     desc "Stops the haproxy server"
     task :stop, :roles => :haproxy do
-      rsudo "service haproxy stop; exit 0"
+      rsudo "service haproxy stop || true"
     end
     
     desc "Starts the haproxy server"

data/templates/haproxy/config/rubber/role/web_tools/rubber-web-tools-haproxy.html

@@ -0,0 +1,18 @@
+<%
+  @path = "/var/www/rubber-web-tools-haproxy.html"
+%>
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+		<title>Rubber HAProxy Admin</title>
+	</head>
+	<body>
+		<h1>Rubber HAProxy Admin</h1>
+
+    <% rubber_instances.for_role('haproxy').sort {|a,b| a.name <=> b.name }.each do |ic| %>
+        <li><a href="/haproxy_<%= ic.name %>/">HAProxy <%= ic.name %></a></li>
+    <% end %>
+
+	</body>
+</html>

data/{generators/vulcanize/templates → templates}/memcached/config/rubber/common/memcached.yml

@@ -1,10 +1,10 @@
 <%
-  @path = "#{RUBBER_ROOT}/config/memcached.yml"
+  @path = "#{Rubber.root}/config/memcached.yml"
   @additive =  ["# rubber-memcached-start", "# rubber-memcached-end"]
 %>
 
 
-<%= RUBBER_ENV %>:
+<%= Rubber.env %>:
   memory: 256
   fragments: true
   benchmarking: false

data/{lib/generators/vulcanize/templates → templates}/mongodb/config/rubber/deploy-mongodb.rb

@@ -5,26 +5,28 @@ namespace :rubber do
     
     rubber.allow_optional_tasks(self)
     
-    before "rubber:install_packages", "rubber:mongodb:install"
-    after "rubber:install_packages", "rubber:mongodb:setup_paths"
+    before "rubber:install_packages", "rubber:mongodb:setup_apt_sources"
   
-    task :install, :roles => :mongodb do
+    task :setup_apt_sources, :roles => :mongodb do
       # Setup apt sources to mongodb from 10gen
       sources = <<-SOURCES
         deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen
       SOURCES
-      sources.gsub!(/^ */, '')
+      sources.gsub!(/^[ \t]*/, '')
       put(sources, "/etc/apt/sources.list.d/mongodb.list") 
       rsudo "apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10"
     end
-    
-    task :setup_paths, :roles => :mongodb do
-      rubber.sudo_script 'setup_mongodb_paths', <<-ENDSCRIPT
-        if [[ ! -d "#{rubber_env.mongodb_data_dir}" ]]; then
-          mkdir -p #{rubber_env.mongodb_data_dir}
-          chown -R mongodb:mongodb #{rubber_env.mongodb_data_dir}
-        fi
-      ENDSCRIPT
+
+    after "rubber:bootstrap", "rubber:mongodb:bootstrap"
+
+    task :bootstrap, :roles => :mongodb do
+      exists = capture("echo $(ls #{rubber_env.mongodb_data_dir} 2> /dev/null)")
+      if exists.strip.size == 0
+        rubber.update_code_for_bootstrap
+        rubber.run_config(:file => "role/mongodb", :force => true, :deploy_path => release_path)
+
+        restart
+      end
     end
     
     desc <<-DESC
@@ -38,14 +40,15 @@ namespace :rubber do
       Stops the mongodb daemon
     DESC
     task :stop, :roles => :mongodb do
-      rsudo "service mongodb stop"
+      rsudo "service mongodb stop || true"
     end
     
     desc <<-DESC
       Restarts the mongodb daemon
     DESC
     task :restart, :roles => :mongodb do
-      rsudo "service mongodb restart"
+      stop
+      start
     end
     
   end

data/templates/mongodb/config/rubber/role/mongodb/mongodb-defaults.conf

@@ -0,0 +1,5 @@
+<%
+  @path = "/etc/default/mongodb"
+%>
+# give mongo more file handles
+ulimit -n 65536