cluster 0.5.33

data/bin/periodic.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This will run periodically to do appropriate backups and cleanups.
+# run as a crontab so the environment is minimal
+
+export GEM_PATH=$(gem env gempath)
+export CREDENTIALS=${HOME}/.cluster/credentials.yml
+
+mysql_user=
+mysql_host=$(cluster service -d database)
+mysql_password=
+aws_secret=""
+aws_key=""
+region=""
+volume=""
+mount=""
+
+cluster period
+
+sudo ec2-consistent-snapshot --aws-access-key-id ${aws_key} --aws-secret-access-key ${aws_secret} --mysql-user ${mysql_user} --mysql-password ${mysql_password} --mysql-host ${mysql_host} --xfs-filesystem ${mount} --region ${region} ${volume}

data/examples/cacerts.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx
+FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD
+VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy
+dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t
+MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB
+MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG
+A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp
+b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl
+cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv
+bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE
+VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ
+ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR
+uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG
+9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI
+hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM
+pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg==
+-----END CERTIFICATE-----

data/examples/credentials.yml

@@ -0,0 +1,24 @@
+amazon:
+  user: 91*********
+  key: AKI**********
+  secret: U***************************
+
+monitor:
+  email:
+    address: smtp.gmail.com
+    port: 587
+    domain: ingamersports.com
+    authentication: plain
+    user_name: dog@score345.com
+    password: *******
+  messages:
+    from: dog@ingamersports.com
+  contacts:
+    - name: alerts
+      email: alert@ingamersports.com
+    - name: panic
+      email: panic@ingamersports.com
+      group: emergency
+    - name: simon
+      email: sdeboer@ingamersports.com
+      group: person

data/examples/monitor.god

@@ -0,0 +1,88 @@
+#!/bin/env god -c
+require 'rubygems'
+require 'tlsmail'
+require 'fileutils'
+require 'cluster'
+require 'cluster/infrastructures/amazon'
+
+Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+
+CREDENTIALS = ENV['CREDENTIALS'] || File.join(ENV['HOME'], '.cluster', 'credentials.yml')
+
+credentials_file = File.expand_path(CREDENTIALS)
+
+unless File.exists?(credentials_file)
+  $stderr.puts "Cannot find credentials for cluster : #{credentials_file} !"
+  exit 2
+end
+MONITOR_PATH = File.join File.dirname(credentials_file), 'services.d'
+
+Cluster::Configuration[:credentials_file] = credentials_file
+CLUSTER = Cluster.new(Amazon.new)
+
+class UserService < God::Behavior
+  def before_start
+    service = self.watch.name[/^(\w+)-/, 1]
+    users_file = File.join(MONITOR_PATH, service, 'users.sh')
+    unless File.exists? users_file
+      FileUtils.mkdir_p File.dirname users_file
+      CLUSTER.retrieve service, 'users.sh', users_file
+    end
+
+    system(users_file) if File.exists? users_file
+  end
+end
+
+class ProjectService < God::Behavior
+  def before_start
+    service = self.watch.name[/^(\w+)-/, 1]
+  end
+end
+
+creds = YAML::load_file credentials_file
+
+if creds.include? 'monitor'
+  moni = creds['monitor']
+  email = moni['email'].keys.inject({}) {|m, k|
+    m.merge k.to_sym => moni['email'][k]
+  }
+  God::Contacts::Email.server_settings = email
+  God::Contacts::Email.message_settings = {
+    :from => moni['messages']['from']
+  }
+
+  if moni.include? 'contacts'
+    for contact in moni['contacts']
+      God.contact(:email) {|c|
+        c.name = contact['name']
+        c.email = contact['email']
+        contact['group'] and (c.group = contact['group'])
+      }
+    end
+  else   
+    God.contact(:email) {|c|
+      c.name = 'system notification'
+      c.email = moni['messages']['from']
+    }
+  end
+end
+
+CLUSTER.instance_state('configuring')
+
+for service in CLUSTER.current('services')
+  monitor_file = File.join MONITOR_PATH, service, "config.god"
+  FileUtils.mkdir_p File.dirname(monitor_file)
+  begin
+    CLUSTER.retrieve service, 'config.god', monitor_file
+  rescue => err
+    applog(nil, :warn, "Service of #{service} does not appear to have a monitor configuration.  Skipping!")
+    next
+  end
+  
+  if File.readable? monitor_file
+    applog(nil, :info, "Loading monitor file #{monitor_file}")
+    load monitor_file
+  else
+    applog(nil, :warn, "Cannot find a monitor file for #{monitor_file}")
+  end
+end

data/examples/users.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if ! id staging 2>/dev/null 1>&2; then
+  adduser --disabled-password --gecos 'staging' staging
+  mkdir -m 700 ~staging/.ssh
+  touch ~staging/.ssh/authorized_keys
+  chmod 600 ~staging/.ssh/authorized_keys
+  touch ~staging/.ssh/config
+  chmod 600 ~staging/.ssh/config
+
+  cat >>~staging/.ssh/authorized_keys <<END_OF_KEYS
+ssh-dss AAAAB3NzaC1kc3MAAACBANcFLeySRpmVji+g9KcHBaedFE5SOLgkSQIKbeVjOwzGC75AIM5gY6gjOu0Kj4BYhlxUwOWkOEYgZwqu15qxtKPCnYQST3jqQnbfwN8UOC4y7XH/4G0gdOD9taFx4PpW0H+zvYs7smBb1qG8NnlUH1tyCGdNNmZLxn2b84R5pDcVAAAAFQDqV2zGPugaRR7gslhxyxrZnIpE9QAAAIAtEViFMBq8h4TXXbdeYR3EhR+zcUHzRz/yB3N7mdo8C1a5apHAnzwX7RPzvkskThU2Idj3ZxUyrTbd5WpBfk/OIwF+hvM3/Uw6XJLNaDSTd0t0HOJxE+SJuHwfOwNtI+J2IXwckUJ2laOTwjOc0YzIR1Gnf50gcz67R07H4qidtQAAAIANQsT7jT68TemlxBjjK22YDRxdRC5M2VuTAT6dDp9JL7KB6B1b3kU7EjiG9gk9oDUGWQiE5sEtHaFjgeViSwQhtaQ/Rqx8EmNG5W9lNgVcWtPmxqMJtOaaNggE+znE1xanY+fDtLqbAkbExZ8vypyUuMXvsgl7AiSSEMnpmX1Itw== simon@s345
+ssh-dss AAAAB3NzaC1kc3MAAACBAIxHXUznZLryPsnK0YCa7W4HM3wSqpCV8MwIBYd4kl/qQzC2VlyX0mFhVFC3esLWyyWEAFcKyI3FlzI70Z4ppnukyL/2MonVTqto3RPoTQs8Ty3y10MYo+9UE65U6Rv2yHfx551vobJgHgVS7tmupCcemZYscgFCeUMgLOz+5tylAAAAFQC9NTxN10PfjndbLDaPYrpC3h8g9wAAAIBnq0vCE9FqjukltP85M5qiawS5sEXArB0HRP4MDpwk4xDdrpiEwa2qYSOiwN1brsxMGrVHe3HetPeApmZZV5vysV1sLp3CQAlGHoWJ3GjOR3lDYpTymlbZOzd62mKp7FxPVtNszbDiABJckTLUV95TM2OB2hn/mWqpzwJBh9GBrAAAAIAmFDbo3aCg3WcDFnBzI72aHBQ4532xEMrlASABJ9FwkNPCtr2VZRLR4vxo9Oyn21+z4SmFocJN0DcKgj1APeS8en/frAFG/UJr/4KdV/LAnTBz8qd6QnbxdUYxizVoYKI1Pl76axbPQCVe+dthfn0x19quOMc/ofHA7I0GGIojJw== nick@Sparta-MB.local
+ssh-dss AAAAB3NzaC1kc3MAAACBAO/mj7W3wUuDAhvicx+C2MkAFJGFZvaPrBA+nbFbK240HDZcmS8sA9Ww3VwivKzQ4CwrpdMY3dPg0TM9H2USQvfMsis4bStDwLy4CFIvBEGoviVVdu8gBzVN+YGajhv7yZqJ9cpOSnUXUzCBh4ddhLtni4IQ2lEZyKdlUzmA77pZAAAAFQDSdiFXpCu3CrAm0zivwA68DcDj5wAAAIEA557V4w7vNgmZvkb3hnQui5IkVTpN+b2PKCzvA9clbJQvKBpivS7k5jpx+GeoAYA+pYNLppMf9zpyPLrJjJhJ3Y3qS3p5iFCed2aJRwxhd5lYIyEedZ2uW6tchVRPoXkT1/tmXBKUNnue8YHslhqXOy8nHyHd7dH7BXLblnQknRcAAACBANzpDp5oXEG7w40tCYd7dtJfDLT8cIXyA75yX2MtkHjpBylT+lP/c1P9pmPytEtBfoQCvwFKjVA7fVGTkF1QfMVzcJsB1wh6jUyS287JtbS4D2O/yEP6FioG5Qva8GdJ29Oy49+LIHgamc93ZaJPub+wvFYivNzQsomjdXtiNRNA olpc@xo-10-F8-44.localdomain
+ssh-dss AAAAB3NzaC1kc3MAAACBAOC9i08N/CoZO9I4kXv8LFSkReglZw9gmOqK72E/IpijLZkd2oh0wyV98S+8lYZR0OPmK6B4vbPIarBsZiPfOe2EMGxsuYjwD3X3S2hIM/1zrvk+WMzf6UADxmCwWzC6bYa1aBH59ba9ojolsBuou6yOxm3boYmZYbnMCLUzm/eLAAAAFQCHNIOpZTKG0chczg+5T3WJ4Q5JAwAAAIA2/VIk8pST/ynuXuu82Z9ZsR9ZJHICYfYi2EqLA6qJUxSB8MjvF9BQGYgX+UCs96uFme8wPJMBU74c7mZlcHvtlgc+1rMQUTZhJ820b/eg2gcPKGYikWj8y3wT0TDoWemMWAaFprvgrPLhHYuVMUKKzI9VNTeKDD7NC1xMsd9pOgAAAIBT14VMwBZjaq0oJSDxzy7bgf5Tragy9yhs7ZgQ0QTpMTFrkO/YvbsCLeXbsRcq904EMcB5glOHXONppn5skwWwBCFtjpjL9WLaeCnhpCxFC+D89GyR7T3Jlvj/Wa2tkuCSFJLBc64kK+pTmDseRLBbhazJazn3oUrNuEFtazzZCw== simon@secondary
+ssh-dss AAAAB3NzaC1kc3MAAACBALlZtmMckFufyVq+It10p4pC65NJCRdDpRrf1uIoVXvz1ZSlsm/f0jhatuNPpwnSQoRpvlWetiS5mzGJ3ze8aReqHHe9CZvDCpVy33cwyRhCZf7ndyGh1qa9ZA0YMHNVdVs03SpNr7Fzc3muGcKX3BiiOZz+6mtbaQW0X1p/VHpLAAAAFQC8AuBVqCRJKgBMHDHb0nxQAyHbRQAAAIEAgQJvHZ3Db3yVy99goeIJw/3lysVemaddIdPuJj/AdxRwkDeuZIxxtOUne/qjXr7hmu48EyXLVg/jsNivKarLI6CXZAk5JbxkgLN+8y0Ckzcd48KhSGpZsbTBtjV29UFH9aiMVOUTbL59603HNa3ucH2lCLhllBf+ameZZp+EFYMAAACARZwlCuPobEAk7wOxWzGQcaRvmSDSbIKexMeUBgZBZeYjp5Bxld6e7x/j3GBQEkzmxqf08zP6LpglDoNR3KuY2TEWhAS3H1KdT4GibNSNpY4y2Pl2K/GjUWC6wMmVcFFBCz/jJorttMSI1G+OVcy+Vdw42HyQ5Z2QtuoVA2MQ5IQ= staging@domU-12-31-39-0E-C8-82
+END_OF_KEYS
+  cat >>~staging/.ssh/config <<END_OF_CONFIG
+Host web
+        HostName domU-12-31-39-00-64-E4.compute-1.internal
+        User app
+Host webu
+        HostName domU-12-31-39-00-64-E4.compute-1.internal
+        User ubuntu
+Host *.compute-1.internal
+        ForwardAgent yes
+END_OF_CONFIG
+
+  chown -R staging:staging ~staging/.ssh
+fi
+
+if ! id app 2>/dev/null 1>&2; then
+  adduser --disabled-password --gecos 'app' app
+  mkdir -m 700 ~app/.ssh
+  touch ~app/.ssh/authorized_keys
+  chmod 600 ~app/.ssh/authorized_keys
+
+  cat >>~app/.ssh/authorized_keys <<END_OF_KEYS
+ssh-dss AAAAB3NzaC1kc3MAAACBALlZtmMckFufyVq+It10p4pC65NJCRdDpRrf1uIoVXvz1ZSlsm/f0jhatuNPpwnSQoRpvlWetiS5mzGJ3ze8aReqHHe9CZvDCpVy33cwyRhCZf7ndyGh1qa9ZA0YMHNVdVs03SpNr7Fzc3muGcKX3BiiOZz+6mtbaQW0X1p/VHpLAAAAFQC8AuBVqCRJKgBMHDHb0nxQAyHbRQAAAIEAgQJvHZ3Db3yVy99goeIJw/3lysVemaddIdPuJj/AdxRwkDeuZIxxtOUne/qjXr7hmu48EyXLVg/jsNivKarLI6CXZAk5JbxkgLN+8y0Ckzcd48KhSGpZsbTBtjV29UFH9aiMVOUTbL59603HNa3ucH2lCLhllBf+ameZZp+EFYMAAACARZwlCuPobEAk7wOxWzGQcaRvmSDSbIKexMeUBgZBZeYjp5Bxld6e7x/j3GBQEkzmxqf08zP6LpglDoNR3KuY2TEWhAS3H1KdT4GibNSNpY4y2Pl2K/GjUWC6wMmVcFFBCz/jJorttMSI1G+OVcy+Vdw42HyQ5Z2QtuoVA2MQ5IQ= staging@domU-12-31-39-0E-C8-82
+END_OF_KEYS
+  chown -R app:app ~app/.ssh
+fi

data/lib/cluster.rb

@@ -0,0 +1,267 @@
+%w(configuration version infrastructure instance release).each {|l| require File.join('cluster', l) }
+
+class Cluster
+  def security(*groups)
+    groups << 'access' if groups.empty?
+    @sub.security(groups)
+  end
+
+  def cost(*sizes)
+    @sub.cost(sizes)
+  end
+
+  def period(*args)
+    @sub.period(args)
+  end
+
+  def revoke(*ips)
+    if ips.empty?
+      current_ip = ''
+      require 'open-uri'
+      open('http://checkip.dyndns.com').each do |line|
+              current_ip = $1 and break if line =~ /IP Address: ([\d\.]+)</
+      end
+      ips << "#{current_ip}/32"
+    end
+    @sub.revoke(ips)
+  end
+
+  def authorize(*ips)
+    if ips.empty?
+      current_ip = ''
+      require 'open-uri'
+      open('http://checkip.dyndns.com').each do |line|
+              current_ip = $1 and break if line =~ /IP Address: ([\d\.]+)</
+      end
+      ips << "#{current_ip}/32"
+    end
+    @sub.authorize(ips)
+  end
+
+  def initialize(infrastructure)
+    self.class.set_credentials_file
+    infrastructure.configure
+    @sub = infrastructure
+  end
+
+  def machine(groups = [])
+    res = machines(groups)
+    res.empty? ? nil : res.first
+  end
+
+  def machines(groups = [])
+    @sub.machines(groups)
+  end
+  alias :instances :machines
+
+  def service(roles)
+    res = services(roles)
+    res.empty? ? nil : res.first
+  end
+
+  def services(roles = [])
+    @sub.services(roles)
+  end
+
+  def labeled(name)
+    @sub.instances.select {|i| i.identified_by? name}
+  end
+
+  def release(*params)
+    klass = @sub.release_class
+    env, tag = params
+    if tag
+      klass.find(env, tag) or klass.create(:environment => env, :tag => tag)
+    elsif env
+      klass.current(env)
+    else
+      nil
+    end
+  end
+
+  def to_credentials
+    {'credentials' => nil}
+  end
+
+  def create_file_store(name)
+    @sub.create_file_store(name)
+  end
+
+  def create_data_store(name)
+    @sub.create_data_store(name)
+  end
+
+  def start(machine_size, *services)
+    size = machine_size.to_s.strip.downcase
+    unless @sub.class.machine_sizes.include? size
+      msg = "#{Cluster::NAME} needs to have a size from: #{@sub.class.machine_sizes.join(', ')}"
+      puts msg
+      raise msg
+    end
+    # FIXME need a way to pass through a number argument
+    number ||= 1
+
+    services = services.map {|s|
+      s.split(',')
+    }.flatten
+
+    # This may not be necessary, but might as well make sure they are
+    # up to date any time that we are starting a new instance.
+    self.save_credentials
+
+    begin
+      number.times.map do |n|
+        @sub.new_instance(size, services)
+      end
+    rescue => err
+      puts "#{Cluster::NAME} cannot start new instance: #{err.message}\n\t#{err.backtrace.join("\n\t")}"
+      exit 2
+    end
+  end
+
+  def update_image_file(filename)
+    input = File.open(filename)
+    @sub.save_images(input)
+  end
+
+  def current(*params)
+    unless @sub.in_cluster?
+      puts "#{Cluster::NAME} says that we aren't in the cluster?"
+      exit 2
+    end
+
+    unless params.length > 0
+      puts "#{Cluster::NAME} current needs to know what to do?\n\t* services\n\t* name (|| dns)\n\t* ip\n\t* enable\n\t* disable"
+      exit 2
+    end
+
+    cmd = params.shift
+
+    case cmd.downcase
+    when 'services'
+      @sub.current_instance.services
+    when /^(n|dn)/
+      @sub.current_instance.dns
+    when /^i/
+      @sub.current_instance.ip
+    when /^e/
+     @sub.alter_instances!(@sub.current_instance) {|i| i.enable *params }
+    when /^di/
+     @sub.alter_instances!(@sub.current_instance) {|i| i.disable *params }
+    when /^s/
+     @sub.alter_instances!(@sub.current_instance) {|i| i.set_state *params }
+    else
+      puts "${Cluster::NAME} current did not understand '#{params.join(' ')}'"
+      exit 1
+    end
+  end
+  alias :instance :current
+
+  def instance_state(state)
+    @sub.current_instance.set_state(state)
+  end
+
+  def store(service, key, filename = nil)
+    full_key = File.join service, key
+    file = if filename and File.readable? filename
+             File.open(filename, 'r')
+           elsif File.readable? full_key
+             File.open(full_key, 'r')
+           elsif File.readable? File.basename(full_key)
+             File.open(File.basename(full_key))
+           else
+             puts "#{Cluster::NAME} cannot open a file for storage"
+             exit 2
+           end
+    @sub.store full_key, file
+  end
+
+  def retrieve(service, key, output = nil)
+    res = @sub.retrieve File.join(service, key)
+    if output
+      begin
+        File.open(output, 'w') {|f|
+          f.write res
+        }
+      rescue => err
+        puts "#{Cluster::NAME} cannot open #{output} for writing."
+        exit 2
+      end
+    else
+      res
+    end
+  end
+
+  def save_credentials(location = nil)
+    creds = if Cluster::Configuration.credentials?
+              File.open(Cluster::Configuration[:credentials_file])
+            else
+              cx = @sub.to_credentials.merge to_credentials
+              StringIO.new(cx.to_yaml)
+            end
+    @sub.save_credentials(creds, location)
+  end
+
+  def save_monitor(filename, key = nil)
+    file = open(filename)
+    unless file
+      puts "#{Cluster::NAME} cannot open file '#{filename}' for reading."
+      exit 2
+    end
+
+    key ||= File.basename(filename)
+    begin
+      @sub.save_monitor file, key
+    rescue => err
+      puts "#{Cluster::NAME} could not save monitor configuration: #{err.message}\n\t#{err.backtrace.join("\n\t")}"
+      exit 2
+    end
+  end
+
+  def credentials_url
+    @sub.credentials_url
+  end
+
+  def fetch_credentials(url)
+    unless Cluster::Configuration.credentials?
+      puts "Need to know where to save the incoming credentials."
+      exit 2
+    end
+
+    out = File.open(Cluster::Configuration[:credentials_file], 'w')
+    open(url).each do |l| out.write(l); end
+    out.close
+  end
+
+  def fetch_monitor(output)
+    monitor = @sub.fetch_monitor
+    unless monitor
+      puts "#{Cluster::NAME} cannot find any monitor information."
+      exit 1
+    end
+
+    File.open(output, 'w') {|f|
+      f.write(monitor)
+    }
+  end
+
+  def gemurl
+    Cluster::LOCATION
+  end
+
+  def imageurl
+    Cluster::IMAGES
+  end
+
+  class << self
+    def set_credentials_file
+      unless Cluster::Configuration.credentials?
+        if ENV['CREDENTIALS'] and File.exist?(ENV['CREDENTIALS'])
+          Cluster::Configuration[:credentials_file] = ENV['CREDENTIALS']
+        elsif ENV['HOME'] and File.exist?(File.join(ENV['HOME'], '.cluster', 'credentials.yml'))
+          Cluster::Configuration[:credentials_file] = File.join(ENV['HOME'], '.cluster', 'credentials.yml')
+        end
+      end
+    end
+  end
+end