cluster 0.6.5 → 0.7.0

data/.gitignore

@@ -0,0 +1,6 @@
+cluster-*.gem
+pkg/
+Session.vim
+.DS_Store
+.*.sw?
+._.DS_Store

data/Rakefile

@@ -4,66 +4,6 @@ require File.join(File.dirname(__FILE__), 'lib', PKG_NAME, 'version')
 namespace :cluster do
   desc "Display #{Cluster::NAME} current version."
   task :version do
-    puts "#{Cluster::NAME} is at version #{Cluster::Version::CURRENT}"
-  end
-
-  desc "Update the #{Cluster::NAME}.gem file at S3."
-  task :update => :package do
-    # Using right_aws directly to get around having to
-    # have a fantastic require structure to use the
-    # cluster tool itself to do this.
-    #
-    # Most particularly this means that error messages will
-    # show up...
-    require 'right_aws'
-
-    Rightscale::HttpConnection.params = Rightscale::HttpConnection.params.merge(:ca_file => File.expand_path(File.join(File.dirname(__FILE__), 'lib', 'cluster', 'infrastructures', 'aws-certificates.crt')))
-
-    oven = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'], ENV['AMAZON_SECRET_ACCESS_KEY']).bucket(Cluster::BUCKET)
-
-    unless oven
-      STDERR.puts "Bucket by the name of #{Cluster::BUCKET} cannot be found or accessed."
-      exit 1
-    end
-
-    filename = File.join('pkg', "#{Cluster::NAME}-#{Cluster::Version::STRING}.gem")
-    file = open(filename)
-    oven.put("#{Cluster::NAME}.gem", file, {}, 'public-read')
-  end
-end
-
-# FIXME these RightAws calls could be refactored into one.
-# Not only that but they should probably be using the cluster
-# infrastructure...
-namespace :images do
-  desc "List the current images used"
-  task :show do
-    require 'right_aws'
-
-    oven = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'], ENV['AMAZON_SECRET_ACCESS_KEY']).bucket(Cluster::BUCKET)
-
-    unless oven
-      STDERR.puts "Bucket by the name of #{Cluster::BUCKET} cannot be found or accessed."
-      exit 1
-    end
-
-    filename = 'cluster_images.yml'
-    puts oven.get(filename)
-  end
-
-  desc "Update the ec2 base image data"
-  task :update do
-    require 'right_aws'
-
-    oven = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'], ENV['AMAZON_SECRET_ACCESS_KEY']).bucket(Cluster::BUCKET)
-
-    unless oven
-      STDERR.puts "Bucket by the name of #{Cluster::BUCKET} cannot be found or accessed."
-      exit 1
-    end
-
-    filename = 'cluster_images.yml'
-    file = open(filename)
-    oven.put(filename, file, {}, 'public-read')
+    puts "#{Cluster::NAME} is at version #{Cluster.version}"
   end
 end

data/cluster.gemspec

@@ -0,0 +1,25 @@
+PKG_NAME = ENV['PKG_NAME'] || 'cluster'
+require File.join(File.dirname(__FILE__), 'lib', PKG_NAME, 'version')
+
+Gem::Specification.new do |gem|
+  gem.version         = Cluster.version.to_s
+  gem.name            = Cluster::NAME
+  gem.executables     = [Cluster::NAME]
+  gem.summary         = %Q{Cluster control - manage many machines in many roles}
+  gem.description     = %Q{You need to scale, you need control, you need feedback, you need monitoring.  You get clustered!}
+  gem.email           = [ Cluster::EMAIL ]
+  gem.authors         = [ Cluster::AUTHOR ]
+  gem.has_rdoc        = false
+
+  gem.files           = `git ls-files`.split("\n")
+  gem.require_paths   = ['lib']
+
+  gem.add_dependency "uuidtools", ">=2.1.1"
+  gem.add_dependency "right_aws"
+  # gem.add_dependency "ingamer_raws", ">= 2.1.0"
+  # gem.add_dependency "rake", ">=0.8.7"
+  # gem.add_dependency "god", ">=0.10.1"
+  # gem.add_dependency "capistrano",       ">=2.5.18"
+  # gem.add_dependency "capistrano-ext",   ">=1.2.1"
+  # gem.add_dependency "railsless-deploy", ">=1.0.0"
+end

data/cluster_images.yml

@@ -0,0 +1,2 @@
+sixtyfour: 'ami-82c913eb'
+name64: '854365875941/base_x86_64_54'

data/etc/cluster_images.yml

@@ -0,0 +1,2 @@
+sixtyfour: ami-c7907eae
+thirtytwo: ami-c5907eac

data/examples/cacerts.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy
+dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t
+MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB
+MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG
+A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp
+b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl
+cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv
+bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE
+VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ
+ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR
+uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG
+9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI
+hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM
+pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg==
+-----END CERTIFICATE-----

data/examples/credentials.yml

@@ -0,0 +1,24 @@
+amazon:
+  user: 91*********
+  key: AKI**********
+  secret: U***************************
+
+monitor:
+  email:
+    address: smtp.gmail.com
+    port: 587
+    domain: ingamersports.com
+    authentication: plain
+    user_name: dog@score345.com
+    password: *******
+  messages:
+    from: dog@ingamersports.com
+  contacts:
+    - name: alerts
+      email: alert@ingamersports.com
+    - name: panic
+      email: panic@ingamersports.com
+      group: emergency
+    - name: simon
+      email: sdeboer@ingamersports.com
+      group: person

data/examples/monitor.god

@@ -0,0 +1,88 @@
+#!/bin/env god -c
+require 'rubygems'
+require 'tlsmail'
+require 'fileutils'
+require 'cluster'
+require 'cluster/infrastructures/amazon'
+
+Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+
+CREDENTIALS = ENV['CREDENTIALS'] || File.join(ENV['HOME'], '.cluster', 'credentials.yml')
+
+credentials_file = File.expand_path(CREDENTIALS)
+
+unless File.exists?(credentials_file)
+  $stderr.puts "Cannot find credentials for cluster : #{credentials_file} !"
+  exit 2
+end
+MONITOR_PATH = File.join File.dirname(credentials_file), 'services.d'
+
+Cluster::Configuration[:credentials_file] = credentials_file
+CLUSTER = Cluster.new(Amazon.new)
+
+class UserService < God::Behavior
+  def before_start
+    service = self.watch.name[/^(\w+)-/, 1]
+    users_file = File.join(MONITOR_PATH, service, 'users.sh')
+    unless File.exists? users_file
+      FileUtils.mkdir_p File.dirname users_file
+      CLUSTER.retrieve service, 'users.sh', users_file
+    end
+
+    system(users_file) if File.exists? users_file
+  end
+end
+
+class ProjectService < God::Behavior
+  def before_start
+    service = self.watch.name[/^(\w+)-/, 1]
+  end
+end
+
+creds = YAML::load_file credentials_file
+
+if creds.include? 'monitor'
+  moni = creds['monitor']
+  email = moni['email'].keys.inject({}) {|m, k|
+    m.merge k.to_sym => moni['email'][k]
+  }
+  God::Contacts::Email.server_settings = email
+  God::Contacts::Email.message_settings = {
+    :from => moni['messages']['from']
+  }
+
+  if moni.include? 'contacts'
+    for contact in moni['contacts']
+      God.contact(:email) {|c|
+        c.name = contact['name']
+        c.email = contact['email']
+        contact['group'] and (c.group = contact['group'])
+      }
+    end
+  else   
+    God.contact(:email) {|c|
+      c.name = 'system notification'
+      c.email = moni['messages']['from']
+    }
+  end
+end
+
+CLUSTER.instance_state('configuring')
+
+for service in CLUSTER.current('services')
+  monitor_file = File.join MONITOR_PATH, service, "config.god"
+  FileUtils.mkdir_p File.dirname(monitor_file)
+  begin
+    CLUSTER.retrieve service, 'config.god', monitor_file
+  rescue => err
+    applog(nil, :warn, "Service of #{service} does not appear to have a monitor configuration.  Skipping!")
+    next
+  end
+  
+  if File.readable? monitor_file
+    applog(nil, :info, "Loading monitor file #{monitor_file}")
+    load monitor_file
+  else
+    applog(nil, :warn, "Cannot find a monitor file for #{monitor_file}")
+  end
+end

data/examples/services.d/memcache/config.god

@@ -0,0 +1,64 @@
+class MemcachedConfig < God::Behavior
+	def before_start
+    service = self.watch.name[/^(\w+)-?/, 1]
+    memcache_dir = File.join(MONITOR_PATH, service)
+
+    memcache_script = File.join(memcache_dir, service)
+    unless File.exists? memcache_script
+      FileUtils.mkdir_p memcache_dir unless File.exists? memcache_dir
+      CLUSTER.retrieve service, service, memcache_file
+    end
+
+    if File.exists? memcache_file
+      system "cd #{memcache_dir} && source #{memcache_file}"
+    end
+  end
+
+	def after_stop
+	    service = self.watch.name[/^(\w+)-?/, 1]
+		CLUSTER.current('disable', service)
+	end
+end
+
+God.watch do |w|
+	w.name = 'memcache'
+	w.interval = 120.seconds
+    memcache_script = File.join(MONITOR_PATH, 'memcache', 'memcached')
+	
+	w.behavior(:memcache_config)
+	w.behavior(:clear_pid_file)
+
+	w.start = memcache_script
+	w.stop = 'killall memcached'
+
+  w.transition([:start, :restart], :up) {|on|
+    on.condition(:process_running) {|c|
+      c.running = true
+      c.notify = 'simon'   
+    } 
+
+    on.condition(:tries) {|c|
+      c.times = 5
+      c.transition = :start
+    }
+  }
+
+  w.transition(:up, :start) {|on|
+    on.condition(:process_exits) {|c|
+      c.notify = 'simon'
+	}
+  }
+
+
+  w.lifecycle {|on|
+    on.condition(:flapping) {|c|
+      c.to_state = [:start, :restart]
+      c.times = 5
+      c.within = 5.minutes
+      c.transition = :unmonitored
+      c.retry_in = 10.minutes
+      c.retry_times = 5
+      c.retry_within = 2.hours
+    }
+  }
+end

data/examples/services.d/memcache/memcached

@@ -0,0 +1,12 @@
+#!/bin/sh
+creds="/home/${USER:-app}/.cluster/credentials.yml"
+if [[ -e $creds ]]; then
+	machine=`cluster -c${creds} current dns`
+fi
+
+if [[ $machine == '' ]]; then
+	machine=`hostname -f`
+fi
+pid=/home/${USER:-app}/checkpoint/shared/pids
+/usr/bin/memcached -d -P $pid -m 600 -l $machine 
+cluster current enable memcache

data/examples/services.d/smtp/main.cf

@@ -0,0 +1,55 @@
+# See /usr/share/postfix/main.cf.dist for a commented, more complete version
+
+# Debian specific:  Specifying a file name will cause the first
+# line of that file to be used as the name.  The Debian default
+# is /etc/mailname.
+#myorigin = /etc/mailname
+
+smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+biff = no
+
+# appending .domain is the MUA's job.
+append_dot_mydomain = no
+
+# Uncomment the next line to generate "delayed mail" warnings
+#delay_warning_time = 4h
+
+readme_directory = no
+
+# TLS parameters
+smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for
+# information on enabling SSL in the smtp client.
+
+myhostname = domU-12-31-38-06-C0-72.compute-1.internal
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+myorigin = /etc/mailname
+mydestination =
+relayhost = [smtp.gmail.com]:587
+#mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+
+# FIXME this should be reset automatically using
+#mynetworks = hash:/etc/postfix/network_table
+mynetworks_style = class
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+default_transport = smtp
+relay_transport = smtp
+inet_protocols = all
+
+# Added for sasl / gmail forwarding
+# smtpd_tls_auth_only was changed from yes (and above)
+smtpd_tls_auth_only = no
+smtpd_use_tls=no
+smtp_use_tls = yes
+smtp_sasl_auth_enable = yes
+smtp_sasl_password_maps = hash:/etc/postfix/sasl/passwd
+smtp_sasl_security_options = noanonymous
+smtp_sasl_tls_security_options = noanonymous
+smtp_tls_CAfile = /etc/postfix/cacerts.pem

data/examples/services.d/web/apache.d/balanced

@@ -0,0 +1,29 @@
+<VirtualHost *:3080>
+        ServerAdmin system@checkpoint.com
+	ServerName www.checkpointevents.com
+	ServerAlias *.checkpointevents.com
+        
+        DocumentRoot /home/app/checkpoint/current/public
+        <Directory />
+                Options FollowSymLinks
+                AllowOverride None
+        </Directory>
+        <Directory /home/app/checkpoint/current/public>
+                Options Indexes FollowSymLinks MultiViews
+                AllowOverride All
+                Order allow,deny
+                allow from all
+        </Directory>
+
+        ErrorDocument 503 /system/maintenance.html
+        RewriteEngine On
+        RewriteCond %{REQUEST_URI} !.(css|gif|jpg|png)$
+        RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f
+        RewriteCond %{SCRIPT_FILENAME} !maintenance.html
+        RewriteRule ^.*$  -  [redirect=503,last]
+
+        LogLevel warn
+
+        ErrorLog /home/app/checkpoint/shared/log/balanced_error.log
+        CustomLog /home/app/checkpoint/shared/log/balanced_access.log combined
+</VirtualHost>

data/examples/services.d/web/apache.d/config.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CONF=/etc/apache2
+
+cp ports.conf $CONF
+cp envvars $CONF
+cp redirect $CONF/sites-available
+cp balanced $CONF/sites-available
+cp php.ini /etc/php5/apache2/php.ini
+
+cd $CONF/sites-enabled
+rm *
+ln -sf ../sites-available/redirect 900-redirect
+ln -sf ../sites-available/staging 250-staging
+ln -sf ../sites-available/balanced 299-balanced
+
+cd $CONF/mods-enabled
+ln -sf ../mods-available/rewrite.load .
+ln -sf ../mods-available/php5.load .
+ln -sf ../mods-available/php5.conf .

data/examples/services.d/web/apache.d/envvars

@@ -0,0 +1,18 @@
+# envvars - default environment variables for apache2ctl
+
+# Since there is no sane way to get the parsed apache2 config in scripts, some
+# settings are defined via environment variables and then used in apache2ctl,
+# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc.
+export APACHE_RUN_USER=www-data
+export APACHE_RUN_GROUP=www-data
+export APACHE_PID_FILE=/var/run/apache2.pid
+
+## The locale used by some modules like mod_dav
+export LANG=C
+## Uncomment the following line to use the system default locale instead:
+#. /etc/default/locale
+
+export LANG
+export CLUSTER_DB=`/var/lib/gems/1.8/bin/cluster -c/home/staging/.cluster/credentials.yml service -d database`
+export CLUSTER_MEMCACHE=`/var/lib/gems/1.8/bin/cluster -c/home/staging/.cluster/credentials.yml services -d memcache`
+export CLUSTER_SMTP=`/var/lib/gems/1.8/bin/cluster -c/home/staging/.cluster/credentials.yml service -d smtp`