rubber-jcarnegie 0.0.1

data/lib/generators/vulcanize/templates/cassandra/config/rubber/role/cassandra/cassandra.in.sh

@@ -0,0 +1,60 @@
+<%
+  @path = "#{rubber_env.cassandra_dir}/bin/cassandra.in.sh"
+%>
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cassandra_home=`dirname $0`/..
+
+# The directory where Cassandra's configs live (required)
+CASSANDRA_CONF=$cassandra_home/conf
+
+# This can be the path to a jar file, or a directory containing the
+# compiled classes. NOTE: This isn't needed by the startup script,
+# it's just used here in constructing the classpath.
+cassandra_bin=$cassandra_home/build/classes
+#cassandra_bin=$cassandra_home/build/cassandra.jar
+
+# JAVA_HOME can optionally be set here
+#JAVA_HOME=/usr/local/jdk6
+
+# The java classpath (required)
+CLASSPATH=$CASSANDRA_CONF:$cassandra_bin
+
+for jar in $cassandra_home/lib/*.jar; do
+    CLASSPATH=$CLASSPATH:$jar
+done
+
+# Arguments to pass to the JVM
+JVM_OPTS=" \
+        -ea \
+        -Xms128M \
+        -Xmx1G \
+        -XX:TargetSurvivorRatio=90 \
+        -XX:+AggressiveOpts \
+        -XX:+UseParNewGC \
+        -XX:+UseConcMarkSweepGC \
+        -XX:+CMSParallelRemarkEnabled \
+        -XX:+HeapDumpOnOutOfMemoryError \
+        -XX:SurvivorRatio=128 \
+        -XX:MaxTenuringThreshold=0 \
+        -Dcom.sun.management.jmxremote.port=<%= rubber_env.cassandra_jmx_port %> \
+        -Dcom.sun.management.jmxremote.ssl=false \
+        -Dcom.sun.management.jmxremote.authenticate=false"
+
+ulimit -n 65536

data/lib/generators/vulcanize/templates/cassandra/config/rubber/role/cassandra/cassandra.yaml

@@ -0,0 +1,210 @@
+<%
+  @path = "#{rubber_env.cassandra_dir}/conf/cassandra.yaml"
+  @post = "mkdir -p #{rubber_env.cassandra_data_dir} #{rubber_env.cassandra_commitlog_dir} #{rubber_env.cassandra_log_dir}"
+%>
+# Cassandra storage config YAML
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# explanations of configuration directives.
+
+# name of the cluster
+cluster_name: '<%= rubber_env.app_name %>_<%= Rubber.env %>_cluster'
+
+# Set to true to make new [non-seed] nodes automatically migrate the
+# right data to themselves.
+auto_bootstrap: <%= rubber_instances.for_role("cassandra").size > 1 ? "true" : "false" %>
+
+# authentication backend, implementing IAuthenticator; used to limit keyspace access
+authenticator: org.apache.cassandra.auth.AllowAllAuthenticator
+
+# any IPartitioner may be used, including your own as long as it is on
+# the classpath.  Out of the box, Cassandra provides
+# org.apache.cassandra.dht.RandomPartitioner
+# org.apache.cassandra.dht.OrderPreservingPartitioner, and
+# org.apache.cassandra.dht.CollatingOrderPreservingPartitioner.
+partitioner: org.apache.cassandra.dht.RandomPartitioner
+
+# directories where Cassandra should store data on disk.
+data_file_directories:
+    - <%= rubber_env.cassandra_data_dir %>
+
+# Addresses of hosts that are deemed contact points.
+# Cassandra nodes use this list of hosts to find each other and learn
+# the topology of the ring.  You must change this if you are running
+# multiple nodes!
+<%
+  seeds = rubber_instances.for_role('cassandra_seed')
+  seeds = rubber_instances.for_role('cassandra') if seeds.size == 0
+  seed_hosts = seeds.collect { |i| i.full_name }
+%>
+seeds:
+    <% seed_hosts.each do |host| %>
+      - <%= host %>
+    <% end %>
+
+# Access mode.  mmapped i/o is substantially faster, but only practical on
+# a 64bit machine (which notably does not include EC2 "small" instances)
+# or relatively small datasets.  "auto", the safe choice, will enable
+# mmapping on a 64bit JVM.  Other values are "mmap", "mmap_index_only"
+# (which may allow you to get part of the benefits of mmap on a 32bit
+# machine by mmapping only index files) and "standard".
+# (The buffer size settings that follow only apply to standard,
+# non-mmapped i/o.)
+disk_access_mode: auto
+
+# Unlike most systems, in Cassandra writes are faster than reads, so
+# you can afford more of those in parallel.  A good rule of thumb is 2
+# concurrent reads per processor core.  Increase ConcurrentWrites to
+# the number of clients writing at once if you enable CommitLogSync +
+# CommitLogSyncDelay. -->
+concurrent_reads: 8
+concurrent_writes: 32
+
+# Buffer size to use when performing contiguous column slices.
+# Increase this to the size of the column slices you typically perform
+sliced_buffer_size_in_kb: 64
+
+# TCP port, for commands and data
+storage_port: <%= rubber_env.cassandra_storage_port %>
+
+# Address to bind to and tell other nodes to connect to. You _must_
+# change this if you want multiple nodes to be able to communicate!
+listen_address: <%= rubber_env.full_host %>
+
+# The address to bind the Thrift RPC service to
+rpc_address: <%= rubber_env.full_host %>
+# port for Thrift to listen on
+rpc_port: <%= rubber_env.cassandra_rpc_port %>
+# Whether or not to use a framed transport for Thrift.
+thrift_framed_transport: false
+snapshot_before_compaction: false
+
+# The threshold size in megabytes the binary memtable must grow to,
+# before it's submitted for flushing to disk.
+binary_memtable_throughput_in_mb: 256
+# Number of minutes to keep a memtable in memory
+memtable_flush_after_mins: 60
+# Size of the memtable in memory before it is dumped
+memtable_throughput_in_mb: 64
+# Number of objects in millions in the memtable before it is dumped
+memtable_operations_in_millions: 0.3
+# Buffer size to use when flushing !memtables to disk.
+flush_data_buffer_size_in_mb: 32
+# Increase (decrease) the index buffer size relative to the data
+# buffer if you have few (many) columns per key.
+flush_index_buffer_size_in_mb: 8
+
+column_index_size_in_kb: 64
+row_warning_threshold_in_mb: 512
+
+# commit log
+commitlog_directory: <%= rubber_env.cassandra_commitlog_dir %>
+
+# Size to allow commitlog to grow to before creating a new segment
+commitlog_rotation_threshold_in_mb: 128
+
+# commitlog_sync may be either "periodic" or "batch."
+# When in batch mode, Cassandra won't ack writes until the commit log
+# has been fsynced to disk.  It will wait up to
+# CommitLogSyncBatchWindowInMS milliseconds for other writes, before
+# performing the sync.
+commitlog_sync: periodic
+
+# the other option is "timed," where writes may be acked immediately
+# and the CommitLog is simply synced every commitlog_sync_period_in_ms
+# milliseconds.
+commitlog_sync_period_in_ms: 10000
+
+# Time to wait for a reply from other nodes before failing the command
+rpc_timeout_in_ms: 10000
+
+# time to wait before garbage collecting tombstones (deletion markers)
+gc_grace_seconds: 864000
+
+# endpoint_snitch -- Set this to a class that implements
+# IEndpointSnitch, which will let Cassandra know enough
+# about your network topology to route requests efficiently.
+# Out of the box, Cassandra provides
+# org.apache.cassandra.locator.SimpleSnitch,
+# org.apache.cassandra.locator.RackInferringSnitch, and
+# org.apache.cassandra.locator.PropertyFileSnitch.
+endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
+
+# A ColumnFamily is the Cassandra concept closest to a relational table.
+#
+# Keyspaces are separate groups of ColumnFamilies.  Except in very
+# unusual circumstances you will have one Keyspace per application.
+#
+# Keyspace required parameters:
+# - name: name of the keyspace; "system" and "definitions" are
+#   reserved for Cassandra Internals.
+# - replica_placement_strategy: the class that determines how replicas
+#   are distributed among nodes.  Must implement IReplicaPlacementStrategy.
+#   Out of the box, Cassandra provides
+#   org.apache.cassandra.locator.RackUnawareStrategy and
+#   org.apache.cassandra.locator.RackAwareStrategy.  RackAwareStrategy
+#   place one replica in each of two datacenter, and other replicas in
+#   different racks in one.
+# - replication_factor: Number of replicas of each row
+# - column_families: column families associated with this keyspace
+#
+# ColumnFamily required parameters:
+# - name: name of the ColumnFamily.  Must not contain the character "-".
+# - compare_with: tells Cassandra how to sort the columns for slicing
+#   operations. The default is BytesType, which is a straightforward
+#   lexical comparison of the bytes in each column.  Other options are
+#   AsciiType, UTF8Type, LexicalUUIDType, TimeUUIDType, and LongType.
+#   You can also specify the fully-qualified class name to a class of
+#   your choice extending org.apache.cassandra.db.marshal.AbstractType.
+#
+# ColumnFamily optional parameters:
+# - keys_cached: specifies the number of keys per sstable whose
+#   locations we keep in memory in "mostly LRU" order.  (JUST the key
+#   locations, NOT any column values.) Specify a fraction (value less
+#   than 1) or an absolute number of keys to cache.  Defaults to 200000
+#   keys.
+# - rows_cached: specifies the number of rows whose entire contents we
+#   cache in memory. Do not use this on ColumnFamilies with large rows,
+#   or ColumnFamilies with high write:read ratios. Specify a fraction
+#   (value less than 1) or an absolute number of rows to cache.
+#   Defaults to 0. (i.e. row caching is off by default)
+# - comment: used to attach additional human-readable information about
+#   the column family to its definition.
+# - read_repair_chance: specifies the probability with which read
+#   repairs should be invoked on non-quorum reads.  must be between 0
+#   and 1. defaults to 1.0 (always read repair).
+# - preload_row_cache: If true, will populate row cache on startup.
+#   Defaults to false.
+#
+keyspaces:
+    - name: <%= rubber_env.app_name.capitalize %>
+      replica_placement_strategy: org.apache.cassandra.locator.RackUnawareStrategy
+      replication_factor: 1
+      column_families:
+        - name: Standard1
+          compare_with: BytesType
+
+        - name: Standard2
+          compare_with: UTF8Type
+          read_repair_chance: 0.1
+          keys_cached: 100
+
+        - name: StandardByUUID1
+          compare_with: TimeUUIDType
+
+        - name: Super1
+          column_type: Super
+          compare_with: BytesType
+          compare_subcolumns_with: BytesType
+
+        - name: Super2
+          column_type: Super
+          compare_subcolumns_with: UTF8Type
+          preload_row_cache: true
+          rows_cached: 10000
+          keys_cached: 50
+          comment: 'A column family with supercolumns, whose column and subcolumn names are UTF8 strings'
+
+        - name: Super3
+          column_type: Super
+          compare_with: LongType
+          comment: 'A column family with supercolumns, whose column names are Longs (8 bytes)'

data/lib/generators/vulcanize/templates/cassandra/config/rubber/role/cassandra/log4j-server.properties

@@ -0,0 +1,45 @@
+<%
+  @path = "#{rubber_env.cassandra_dir}/conf/log4j-server.properties"
+%>
+
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set the root to INFO
+# and the pattern to %c instead of %l.  (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=INFO,stdout,R
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%5p %d{HH:mm:ss,SSS} %m%n
+
+# rolling log file
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.file.maxFileSize=20MB
+log4j.appender.file.maxBackupIndex=50
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%5p [%t] %d{ISO8601} %F (line %L) %m%n
+# Edit the next line to point to your logs directory
+log4j.appender.R.File=<%= rubber_env.cassandra_log_dir %>/system.log
+
+# Application logging options
+#log4j.logger.com.facebook=DEBUG
+#log4j.logger.com.facebook.infrastructure.gms=DEBUG
+#log4j.logger.com.facebook.infrastructure.db=DEBUG

data/lib/generators/vulcanize/templates/cassandra/config/rubber/role/cassandra/monit-cassandra.conf

@@ -0,0 +1,8 @@
+<%
+  @path = '/etc/monit/monit.d/monit-cassandra.conf'
+%>
+check process cassandra with pidfile <%= rubber_env.cassandra_pid_file %>
+  group cassandra-<%= RUBBER_ENV %>
+  start program = "<%= rubber_env.cassandra_dir %>/bin/cassandra -p <%= rubber_env.cassandra_pid_file %>"
+  stop program = "kill `cat <%= rubber_env.cassandra_pid_file %>`"
+  if failed host <%= rubber_env.host %> port <%= rubber_env.cassandra_rpc_port %> with timeout 10 seconds for 10 cycles then restart

data/lib/generators/vulcanize/templates/cassandra/config/rubber/rubber-cassandra.yml

@@ -0,0 +1,18 @@
+cassandra_version: 2010-05-05_14-04-54
+cassandra_prefix: /usr/local
+cassandra_dir: "#{cassandra_prefix}/apache-cassandra-#{cassandra_version}"
+cassandra_data_dir: "/var/lib/cassandra/data"
+cassandra_commitlog_dir: "/var/lib/cassandra/commitlog"
+cassandra_log_dir: "/var/log/cassandra"
+cassandra_pid_file: "/var/run/cassandra.pid"
+cassandra_pkg_url: "http://hudson.zones.apache.org/hudson/job/Cassandra/lastSuccessfulBuild/artifact/cassandra/build/apache-cassandra-#{cassandra_version}-bin.tar.gz"
+cassandra_rpc_port: 9160
+cassandra_storage_port: 7070
+cassandra_jmx_port: 8181
+
+role_dependencies:
+  cassandra_seed: [cassandra]
+  
+roles:
+  cassandra:
+    packages: [openjdk-6-jdk]

data/lib/generators/vulcanize/templates/cassandra/templates.yml

@@ -0,0 +1 @@
+description: The Cassandra distributed data store 

data/lib/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/haproxy/haproxy-mongrel.conf

@@ -0,0 +1,23 @@
+<%
+  @path = "/etc/haproxy/haproxy.cfg"
+  @additive = ["# start haproxy mongrel", "# end haproxy mongrel"]
+%>
+
+listen mongrel_proxy 0.0.0.0:<%= rubber_env.haproxy_listen_port %>
+  <%
+    # Since we do "fair" load balancing to the mongrels, we don't want to share mongrels
+    # between haproxy instances otherwise we still end up with requests getting queued behind
+    # slow ones.
+    app_hosts = rubber_instances.for_role('app').collect { |i| i.name }
+    haproxy_hosts = rubber_instances.for_role('haproxy').collect { |i| i.name }
+    ports = (rubber_env.mongrel_base_port.to_i ... rubber_env.mongrel_base_port.to_i + rubber_env.mongrel_count.to_i).to_a
+    host_count = haproxy_hosts.size
+    host_mod = haproxy_hosts.sort.index(rubber_env.host) rescue 0
+    ports = ports.find_all {|x| x % host_count == host_mod}
+  %>
+  <% app_hosts.each do |server| %>
+    <% ports.each do |port| %>
+      server <%= server %>_<%= port %> <%= server %>:<%= port %> maxconn 1 check
+    <% end %>
+  <% end %>
+

data/lib/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/nginx/nginx-mongrel.conf

@@ -0,0 +1,113 @@
+<%
+  @path = "/etc/nginx/rubber/mongrel.conf"
+%>
+
+upstream haproxy_mongrel
+{
+  <% rubber_instances.for_role('haproxy').each do |ic| %>
+    server <%= ic.name %>:<%= rubber_env.haproxy_listen_port %>;
+  <% end %>
+}
+
+
+<% %w[http https].each do |protocol| %>
+
+  # This server is setup to serve www over <%= protocol %>.
+  server
+  {
+    <% if protocol == 'https' %>
+      listen               <%= rubber_env.web_ssl_port %>;
+      ssl                  on;
+      ssl_certificate    /etc/ssl/certs/ssl-cert-snakeoil.pem;
+      ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
+      # ssl_certificate      <%= RUBBER_ROOT %>/config/snapmylife.com.crt;
+      # ssl_certificate_key  <%= RUBBER_ROOT %>/config/snapmylife.com.key;
+    <% else %>
+      listen <%= rubber_env.web_port %>;
+    <% end %>
+
+    client_max_body_size 10M;
+
+    server_name <%= rubber_env.domain %>;
+
+    if ( $host = www.<%= rubber_env.domain %>)
+    {
+      rewrite ^\/(.*)$ http://<%= rubber_env.domain %>/$1 permanent;
+    }
+
+    root <%= RUBBER_ROOT + "/public" %>;
+
+    # this rewrites all the requests to the maintenance.html
+    # page if it exists in the doc root. This is for capistrano's
+    # disable web task
+    if (-f $document_root/system/maintenance.html)
+    {
+      rewrite  ^(.*)$  /system/maintenance.html last;
+      break;
+    }
+
+    location /
+    {
+      <%# nginx -> rails is http, so this tells rails the source protocol was https %>
+      <%= 'proxy_set_header X_FORWARDED_PROTO https;' if protocol == 'https' %>
+    
+      # needed to forward user's IP address to rails
+      proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      # needed for HTTPS
+      proxy_set_header Host $http_host;
+      proxy_redirect false;
+      proxy_max_temp_file_size 0;
+
+
+      # If the file exists as a static file serve it directly without
+      # running all the other rewrite tests on it
+      if (-f $request_filename)
+      {
+        break;
+      }
+
+      # check for index.html for directory index
+      # if its there on the filesystem then rewite
+      # the url to add /index.html to the end of it
+      # and then break to send it to the next config rules.
+      if (-f $request_filename/index.html)
+      {
+        rewrite (.*) $1/index.html break;
+      }
+
+      # this is the meat of the rails page caching config
+      # it adds .html to the end of the url and then checks
+      # the filesystem for that file. If it exists, then we
+      # rewite the url to have explicit .html on the end
+      # and then send it on its way to the next config rule.
+      # if there is no file on the fs then it sets all the
+      # necessary headers and proxies to our upstream mongrels
+      if (-f $request_filename.html)
+      {
+        rewrite (.*) $1.html break;
+      }
+
+      if (!-f $request_filename)
+      {
+        proxy_pass http://haproxy_mongrel;
+        break;
+      }
+    }
+
+    # catch errors from proxy so we can display our error page
+    proxy_intercept_errors on;
+
+    error_page   500 502 503 504  /500.html;
+    location = /500.html
+    {
+      root <%= RUBBER_ROOT + "/public" %>;
+    }
+    error_page 404  /404.html;
+    location = /404.html
+    {
+      root <%= RUBBER_ROOT + "/public" %>;
+    }
+  }
+
+<% end %>