sml-rubber 0.9.1

data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/crontab

@@ -0,0 +1,11 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-sphinx-start", "# rubber-sphinx-end"]
+%>
+
+# incremental index once every 10 minutes
+*/10 * * * * sudo -u rails <%= RUBBER_ROOT %>/script/cron-rake RUBBER_ENV=<%= RUBBER_ENV %> ts:in:delta
+
+# full reindex daily at 4:35
+35 4 * * * sudo -u rails <%= RUBBER_ROOT %>/script/cron-rake RUBBER_ENV=<%= RUBBER_ENV %> ts:index

data/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/monit-sphinx.conf

@@ -0,0 +1,10 @@
+<%
+  @path = '/etc/monit/monit.d/monit-sphinx.conf'
+%>
+
+check process sphinx with pidfile <%= RUBBER_ROOT %>/log/searchd.<%= RUBBER_ENV %>.pid
+  group sphinx-<%= RUBBER_ENV %>
+  start program = "/usr/bin/sudo -u rails sh -c 'cd <%= RUBBER_ROOT %> && PATH=/usr/local/bin:$PATH RAILS_ENV=<%= RUBBER_ENV %> rake ts:start'"
+  stop program  = "/usr/bin/sudo -u rails sh -c 'cd <%= RUBBER_ROOT %> && PATH=/usr/local/bin:$PATH RAILS_ENV=<%= RUBBER_ENV %> rake ts:stop'"
+  if failed host <%= rubber_env.host %> port 3312 with timeout 5 seconds for 5 cycles then restart
+

data/generators/vulcanize/templates/sphinx/config/rubber/rubber-sphinx.yml

@@ -0,0 +1,6 @@
+gems: []
+
+roles:
+  sphinx:
+    packages: [bzr-builddeb, autotools-dev, dpatch, libpq-dev, autoconf,
+               libtool, automake, gawk, libmysqlclient15-dev]

data/generators/vulcanize/templates/sphinx/templates.yml

@@ -0,0 +1 @@
+description: The sphinx searchd / thinking sphinx module

data/generators/vulcanize/vulcanize_generator.rb

@@ -0,0 +1,67 @@
+require 'fileutils'
+require 'find'
+
+class VulcanizeGenerator < Rails::Generator::NamedBase
+  
+  TEMPLATE_ROOT = File.dirname(__FILE__) + "/templates"
+  TEMPLATE_FILE = "templates.yml"
+    
+  def manifest
+    record do |m|
+      templates = [file_name] + actions
+      templates.each do |t|
+        apply_template(m, t)
+      end
+    end
+  end
+
+  def apply_template(m, name)
+    sp = source_path("#{name}/")
+    unless File.directory?(sp)
+      raise Rails::Generator::UsageError.new("Invalid template #{name}, use one of #{valid_templates.join(', ')}")
+    end
+    
+    templ_conf = load_template_config(sp)
+    deps = templ_conf['dependent_templates'] || []
+    deps.each do |dep|
+      apply_template(m, dep)
+    end
+    
+    Find.find(sp) do |f|
+      Find.prune if File.basename(f) =~ /^(CVS|\.svn)$/
+      Find.prune if f == "#{sp}#{TEMPLATE_FILE}"
+      
+      rel = f.gsub(/#{source_root}\//, '')
+      dest_rel = rel.gsub(/^#{name}\//, '')
+      m.directory(dest_rel) if File.directory?(f)
+      if File.file?(f)
+        # force scripts to be executable
+        opts = (File.read(f) =~ /^#!/) ? {:chmod => 0755} : {}
+        m.file(rel, dest_rel, opts)
+      end
+
+    end
+  end
+
+  protected
+    def valid_templates
+      valid = Dir.entries(TEMPLATE_ROOT).delete_if {|e| e =~ /(^\.)|svn|CVS/ }
+    end
+    
+    def load_template_config(template_dir)
+      templ_file = "#{template_dir}/templates.yml"
+      templ_conf = YAML.load(File.read(templ_file)) rescue {}
+      return templ_conf
+    end
+
+    def banner
+      usage = "Usage: #{$0} vulcanize template_name ...\n"
+      usage << "where template_name is one of:\n\n"
+      valid_templates.each do |t|
+        templ_conf = load_template_config("#{TEMPLATE_ROOT}/#{t}")
+        desc = templ_conf['description']
+        usage << "    #{t}: #{desc}\n" 
+      end
+      return usage
+    end
+end

data/lib/capistrano/hostcmd.rb

@@ -0,0 +1,12 @@
+require 'capistrano/command'
+
+# capistrano hack to allow us to run slightly different commands on multiple
+# hosts in parallel
+module Capistrano
+  class Command
+    def replace_placeholders(command, channel)
+      command = command.gsub(/\$CAPISTRANO:HOST\$/, channel[:host])
+      command.gsub(/\$CAPISTRANO:VAR\$/, @options["hostvar_#{channel[:host]}"].to_s)
+    end
+  end
+end

data/lib/rubber.rb

@@ -0,0 +1,37 @@
+$:.unshift(File.dirname(__FILE__))
+
+module Rubber
+
+  def self.initialize(project_root, project_env)
+    return if defined?(RUBBER_ROOT) && defined?(RUBBER_ENV)
+    
+    Object.const_set('RUBBER_ENV', project_env)
+    Object.const_set('RUBBER_ROOT', project_root)
+
+    # pull in basic rails env.  rubber only needs RAILS_ROOT and RAILS_ENV.
+    # We actually do NOT want the entire rails environment because it
+    # complicates bootstrap (i.e. can't run config to create db because full
+    # rails env needs db to exist as some plugin accesses model or something)
+    rails_boot_file = File.join(RUBBER_ROOT, 'config', 'boot')
+    require(rails_boot_file) if File.exists? rails_boot_file
+
+    if defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER
+      Object.const_set('LOGGER', RAILS_DEFAULT_LOGGER)
+    else
+      Object.const_set('LOGGER', Logger.new($stdout))
+      LOGGER.level = Logger::INFO
+      LOGGER.formatter = lambda {|severity, time, progname, msg| "Rubber[%s]: %s\n" % [severity, msg.to_s.lstrip]}
+    end
+  end
+
+  VERSION = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).chomp
+end
+
+
+require 'rubber/configuration'
+require 'rubber/environment'
+require 'rubber/generator'
+require 'rubber/instance'
+require 'rubber/util'
+require 'rubber/cloud'
+require 'rubber/dns'

data/lib/rubber/capistrano.rb

@@ -0,0 +1 @@
+Capistrano::Configuration.instance.load(File.join(File.dirname(__FILE__), 'recipes', 'rubber.rb'))

data/lib/rubber/cloud.rb

@@ -0,0 +1,13 @@
+require "rubber/cloud/base"
+
+module Rubber
+  module Cloud
+
+    def self.get_provider(provider, env, capistrano)
+      require "rubber/cloud/#{provider}"
+      clazz = Rubber::Cloud.const_get(provider.capitalize)
+      return clazz.new(env, capistrano)
+    end
+
+  end
+end

data/lib/rubber/cloud/aws.rb

@@ -0,0 +1,261 @@
+require 'rubygems'
+require 'EC2'
+require 'aws/s3'
+
+module Rubber
+  module Cloud
+
+    class Aws < Base
+
+      def initialize(env, capistrano)
+        super(env, capistrano)
+        @aws_env = env.cloud_providers.aws
+        @ec2 = EC2::Base.new(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+        AWS::S3::Base.establish_connection!(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+      end
+
+
+      def create_instance(ami, ami_type, security_groups, availability_zone)
+        response = @ec2.run_instances(:image_id => ami, :key_name => @aws_env.key_name, :instance_type => ami_type, :group_id => security_groups, :availability_zone => availability_zone)
+        instance_id = response.instancesSet.item[0].instanceId
+        return instance_id
+      end
+
+      def describe_instances(instance_id=nil)
+        instances = []
+
+        response = @ec2.describe_instances(:instance_id => instance_id)
+        response.reservationSet.item.each do |ritem|
+          ritem.instancesSet.item.each do |item|
+            instance = {}
+            instance[:id] = item.instanceId
+            instance[:external_host] = item.dnsName
+            instance[:external_ip] = IPSocket.getaddress(instance[:external_host]) rescue nil
+            instance[:internal_host] = item.privateDnsName
+            instance[:state] = item.instanceState.name
+            instance[:zone] = item.placement.availabilityZone
+            instances << instance
+          end
+        end if response.reservationSet
+
+        return instances
+      end
+
+      def destroy_instance(instance_id)
+        response = @ec2.terminate_instances(:instance_id => instance_id)
+      end
+
+      def describe_availability_zones
+        zones = []
+        response = @ec2.describe_availability_zones()
+        response.availabilityZoneInfo.item.each do |item|
+          zone = {}
+          zone[:name] = item.zoneName
+          zone[:state] =item.zoneState
+          zones << zone
+        end if response.availabilityZoneInfo
+        return zones
+      end
+
+      def create_security_group(group_name, group_description)
+        @ec2.create_security_group(:group_name => group_name, :group_description => group_description)
+      end
+
+      def describe_security_groups(group_name=nil)
+        groups = []
+
+        opts = {}
+        opts[:group_name] = group_name if group_name
+        response = @ec2.describe_security_groups(opts)
+
+        response.securityGroupInfo.item.each do |item|
+          group = {}
+          group[:name] = item.groupName
+          group[:description] = item.groupDescription
+
+          item.ipPermissions.item.each do |ip_item|
+            group[:permissions] ||= []
+            rule = {}
+
+            rule[:protocol] = ip_item.ipProtocol
+            rule[:from_port] = ip_item.fromPort
+            rule[:to_port] = ip_item.toPort
+
+            ip_item.groups.item.each do |rule_group|
+              rule[:source_groups] ||= []
+              source_group = {}
+              source_group[:account] = rule_group.userId
+              source_group[:name] = rule_group.groupName
+              rule[:source_groups] << source_group
+            end if ip_item.groups
+
+            ip_item.ipRanges.item.each do |ip_range|
+              rule[:source_ips] ||= []
+              rule[:source_ips] << ip_range.cidrIp
+            end if ip_item.ipRanges
+
+            group[:permissions] << rule
+          end if item.ipPermissions
+
+          groups << group
+          
+        end if response.securityGroupInfo
+
+        return groups
+      end
+
+      def add_security_group_rule(group_name, protocol, from_port, to_port, source)
+        opts = {:group_name => group_name}
+        if source.instance_of? Hash
+          opts = opts.merge(:source_security_group_name => source[:name], :source_security_group_owner_id => source[:account])
+        else
+          opts = opts.merge(:ip_protocol => protocol, :from_port => from_port, :to_port => to_port, :cidr_ip => source)
+        end
+        @ec2.authorize_security_group_ingress(opts)
+      end
+
+      def remove_security_group_rule(group_name, protocol, from_port, to_port, source)
+        opts = {:group_name => group_name}
+        if source.instance_of? Hash
+          opts = opts.merge(:source_security_group_name => source[:name], :source_security_group_owner_id => source[:account])
+        else
+          opts = opts.merge(:ip_protocol => protocol, :from_port => from_port, :to_port => to_port, :cidr_ip => source)
+        end
+        @ec2.revoke_security_group_ingress(opts)
+      end
+
+      def destroy_security_group(group_name)
+        @ec2.delete_security_group(:group_name => group_name)
+      end
+
+      def create_static_ip
+        response = @ec2.allocate_address()
+        return response.publicIp
+      end
+
+      def attach_static_ip(ip, instance_id)
+        response = @ec2.associate_address(:instance_id => instance_id, :public_ip => ip)
+        return response.return == "true"
+      end
+
+      def detach_static_ip(ip)
+        response = @ec2.disassociate_address(:public_ip => ip)
+        return response.return == "true"
+      end
+
+      def describe_static_ips(ip=nil)
+        ips = []
+        response = @ec2.describe_addresses()
+        response.addressesSet.item.each do |item|
+          ip = {}
+          ip[:instance_id] = item.instanceId
+          ip[:ip] = item.publicIp
+          ips << ip
+        end if response.addressesSet
+        return ips
+      end
+
+      def destroy_static_ip(ip)
+        response = @ec2.release_address(:public_ip => ip)
+        return response.return == "true"
+      end
+
+      def create_volume(size, zone)
+        response = @ec2.create_volume(:size => size.to_s, :availability_zone => zone)
+        return response.volumeId
+      end
+
+      def attach_volume(volume_id, instance_id, device)
+        response = @ec2.attach_volume(:volume_id => volume_id, :instance_id => instance_id, :device => device)
+        return response.status
+      end
+
+      def detach_volume(volume_id)
+        @ec2.detach_volume(:volume_id => volume_id, :force => 'true')
+      end
+
+      def describe_volumes(volume_id=nil)
+        volumes = []
+        opts = {}
+        opts[:volume_id] = volume_id if volume_id
+        response = @ec2.describe_volumes(opts)
+        response.volumeSet.item.each do |item|
+          volume = {}
+          volume[:id] = item.volumeId
+          volume[:status] = item.status
+          if attach = item.attachmentSet.item[0]
+            volume[:attachment_instance_id] = attach.instanceId
+            volume[:attachment_status] = attach.status
+          end
+          volumes << volume
+        end if response.volumeSet
+        return volumes
+      end
+
+      def destroy_volume(volume_id)
+        @ec2.delete_volume(:volume_id => volume_id)
+      end
+
+      def create_image(image_name)
+        ec2_key = @aws_env.key_file
+        ec2_pk = @aws_env.pk_file
+        ec2_cert = @aws_env.cert_file
+        aws_account = @aws_env.account
+        ec2_key_dest = "/mnt/#{File.basename(ec2_key)}"
+        ec2_pk_dest = "/mnt/#{File.basename(ec2_pk)}"
+        ec2_cert_dest = "/mnt/#{File.basename(ec2_cert)}"
+
+        capistrano.put(File.read(ec2_key), ec2_key_dest)
+        capistrano.put(File.read(ec2_pk), ec2_pk_dest)
+        capistrano.put(File.read(ec2_cert), ec2_cert_dest)
+
+        arch = capistrano.capture "uname -m"
+        arch = case arch when /i\d86/ then "i386" else arch end
+        capistrano.sudo_script "create_bundle", <<-CMD
+          export RUBYLIB=/usr/lib/site_ruby/
+          ec2-bundle-vol --batch -d /mnt -k #{ec2_pk_dest} -c #{ec2_cert_dest} -u #{account} -p #{image_name} -r #{arch}
+        CMD
+
+        capistrano.sudo_script "register_bundle", <<-CMD
+          export RUBYLIB=/usr/lib/site_ruby/
+          ec2-upload-bundle --batch -b #{@aws_env.image_bucket} -m /mnt/#{image_name}.manifest.xml -a #{@aws_env.access_key} -s #{@aws_env.secret_access_key}
+        CMD
+
+        image_location = "#{@aws_env.image_bucket}/#{image_name}.manifest.xml"
+        response = @ec2.register_image(:image_location => image_location)
+        return response.imageId
+      end
+
+      def describe_images(image_id=nil)
+        images = []
+        response = @ec2.describe_images(:owner_id => 'self', :image_id => image_id)
+        response.imagesSet.item.each do |item|
+          image = {}
+          image[:id] = item.imageId
+          image[:location] = item.imageLocation
+          images << image
+        end if response.imagesSet
+        return images
+      end
+
+      def destroy_image(image_id)
+        image = describe_images(image_id).first
+        image_location = image[:location]
+        bucket = image_location.split('/').first
+        image_name = image_location.split('/').last.gsub(/\.manifest\.xml$/, '')
+
+        @ec2.deregister_image(:image_id => image_id)
+
+        s3_bucket = AWS::S3::Bucket.find(bucket)
+        s3_bucket.objects(:prefix => image_name).clone.each do |obj|
+          obj.delete
+        end
+        if s3_bucket.empty?
+          s3_bucket.delete
+        end
+      end
+
+    end
+
+  end
+end

data/lib/rubber/cloud/base.rb

@@ -0,0 +1,16 @@
+module Rubber
+  module Cloud
+
+    class Base
+
+      attr_reader :env, :capistrano
+
+      def initialize(env, capistrano)
+        @env = env
+        @capistrano = capistrano
+      end
+
+    end
+
+  end
+end