rubber-jcarnegie 0.0.1

data/lib/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/crontab

@@ -0,0 +1,11 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-sphinx-start", "# rubber-sphinx-end"]
+%>
+
+# incremental index once every 10 minutes
+*/10 * * * * sudo -u <%= rubber_env.app_user %> <%= RUBBER_ROOT %>/script/cron-rake RUBBER_ENV=<%= RUBBER_ENV %> ts:in:delta
+
+# full reindex daily at 4:35
+35 4 * * * sudo -u <%= rubber_env.app_user %> <%= RUBBER_ROOT %>/script/cron-rake RUBBER_ENV=<%= RUBBER_ENV %> ts:index

data/lib/generators/vulcanize/templates/sphinx/config/rubber/role/sphinx/monit-sphinx.conf

@@ -0,0 +1,13 @@
+<%
+  @path = '/etc/monit/monit.d/monit-sphinx.conf'
+
+  PIDFILE = "#{RUBBER_ROOT}/log/searchd.#{RUBBER_ENV}.pid"
+  start_program = "cd #{RUBBER_ROOT} && rake ts:start"
+  stop_program = "cd #{RUBBER_ROOT} && rake ts:stop"
+%>
+
+check process sphinx with pidfile <%= PIDFILE %>
+  group sphinx-<%= RUBBER_ENV %>
+  start program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> bash -l -c '<%= start_program %>'"
+  stop program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> bash -l -c '<%= stop_program %>'"
+  if failed host <%= rubber_env.host %> port 3312 with timeout 5 seconds for 5 cycles then restart

data/lib/generators/vulcanize/templates/sphinx/config/rubber/rubber-sphinx.yml

@@ -0,0 +1,6 @@
+gems: []
+
+roles:
+  sphinx:
+    packages: [bzr-builddeb, autotools-dev, dpatch, libpq-dev, autoconf,
+               libtool, automake, gawk, libmysqlclient15-dev]

data/lib/generators/vulcanize/templates/sphinx/templates.yml

@@ -0,0 +1 @@
+description: The sphinx searchd / thinking sphinx module

data/lib/generators/vulcanize/vulcanize_generator.rb

@@ -0,0 +1,184 @@
+require 'find'
+require 'yaml'
+
+# Load and initialize rubber if it hasn't been already.
+# This happens using the generator with Rails 2 and Rubber is not set up as a plugin.
+unless defined?(Rubber)
+  env = ENV['RUBBER_ENV'] ||= 'development'
+  root = '.'
+
+  require 'rubber'
+  Rubber::initialize(root, env)
+end
+
+if Rubber::Util::is_rails2?
+  require 'fileutils'
+  require 'commands/generate'
+
+  class VulcanizeGenerator < Rails::Generator::NamedBase
+    include Rails::Generator::Commands
+
+    TEMPLATE_ROOT = File.dirname(__FILE__) + "/templates" unless defined?(TEMPLATE_ROOT)
+    TEMPLATE_FILE = "templates.yml" unless defined?(TEMPLATE_FILE)
+
+    def manifest
+      record do |m|
+        templates = [file_name] + actions
+        templates.each do |t|
+          apply_template(m, t)
+        end
+      end
+    end
+
+    def apply_template(m, name)
+      sp = source_path("#{name}/")
+      unless File.directory?(sp)
+        raise Rails::Generator::UsageError.new("Invalid template #{name}, use one of #{valid_templates.join(', ')}")
+      end
+
+      @template_dependencies ||= []
+
+      templ_conf = load_template_config(sp)
+      deps = templ_conf['dependent_templates'] || []
+      @template_dependencies.concat(deps)
+      deps.each do |dep|
+        apply_template(m, dep)
+      end
+
+      extra_generator_steps_file = File.join(TEMPLATE_ROOT, name, 'templates.rb')
+
+      Find.find(sp) do |f|
+        Find.prune if File.basename(f) =~ /^(CVS|\.svn)$/
+        Find.prune if f == "#{sp}#{TEMPLATE_FILE}"
+        Find.prune if f == extra_generator_steps_file # don't copy over templates.rb
+
+        rel = f.gsub(/#{source_root}\//, '')
+        dest_rel = rel.gsub(/^#{name}\//, '')
+
+        # Only include optional files when their conditions eval to true
+        template_conf = YAML.load(File.read(TEMPLATE_FILE)) rescue {}
+        optional = template_conf['optional'][dest_rel] rescue nil
+        Find.prune if optional && ! eval(optional)
+
+        m.directory(dest_rel) if File.directory?(f)
+        if File.file?(f)
+          # force scripts to be executable
+          opts = (File.read(f) =~ /^#!/) ? {:chmod => 0755} : {}
+          m.file(rel, dest_rel, opts)
+        end
+      end
+
+      if File.exist? extra_generator_steps_file
+        eval File.read(extra_generator_steps_file), binding, extra_generator_steps_file
+      end
+    end
+
+    protected
+      def valid_templates
+        valid = Dir.entries(TEMPLATE_ROOT).delete_if {|e| e =~ /(^\.)|svn|CVS/ }
+      end
+
+      def load_template_config(template_dir)
+        templ_file = "#{template_dir}/templates.yml"
+        templ_conf = YAML.load(File.read(templ_file)) rescue {}
+        return templ_conf
+      end
+
+      def banner
+        usage = "Usage: #{$0} vulcanize template_name ...\n"
+        usage << "where template_name is one of:\n\n"
+        valid_templates.each do |t|
+          templ_conf = load_template_config("#{TEMPLATE_ROOT}/#{t}")
+          desc = templ_conf['description']
+          usage << "    #{t}: #{desc}\n"
+        end
+        return usage
+      end
+  end
+
+else
+  require 'rails/generators'
+
+  class VulcanizeGenerator < Rails::Generators::NamedBase
+
+    def self.source_root
+      File.join(File.dirname(__FILE__), 'templates')
+    end
+
+    def copy_template_files
+      @template_dependencies = find_dependencies(file_name)
+      ([file_name] + @template_dependencies).each do |template|
+        apply_template(template)
+      end
+    end
+
+    protected
+
+    # helper to test for rails for optional templates
+    def rails?
+      Rubber::Util::is_rails?
+    end
+
+    def find_dependencies(name)
+      template_dir = File.join(self.class.source_root, name, '')
+      unless File.directory?(template_dir)
+        raise Rails::Generators::Error.new("Invalid template #{name}, use one of #{valid_templates.join(', ')}")
+      end
+
+      template_conf = load_template_config(template_dir)
+      template_dependencies = template_conf['dependent_templates'] || []
+
+      template_dependencies.clone.each do |dep|
+        template_dependencies.concat(find_dependencies(dep))
+      end
+
+      return template_dependencies.uniq
+    end
+
+    def apply_template(name)
+      template_dir = File.join(self.class.source_root, name, '')
+      unless File.directory?(template_dir)
+        raise Rails::Generators::Error.new("Invalid template #{name}, use one of #{valid_templates.join(', ')}")
+      end
+
+      template_conf = load_template_config(template_dir)
+
+      extra_generator_steps_file = File.join(template_dir, 'templates.rb')
+
+      Find.find(template_dir) do |f|
+        Find.prune if f == File.join(template_dir, 'templates.yml')  # don't copy over templates.yml
+        Find.prune if f == extra_generator_steps_file # don't copy over templates.rb
+
+        template_rel = f.gsub(/#{template_dir}/, '')
+        source_rel = f.gsub(/#{self.class.source_root}\//, '')
+        dest_rel   = source_rel.gsub(/^#{name}\//, '')
+
+        # Only include optional files when their conditions eval to true
+        optional = template_conf['optional'][template_rel] rescue nil
+        Find.prune if optional && ! eval(optional)
+
+        if File.directory?(f)
+          empty_directory(dest_rel)
+        else
+          copy_file(source_rel, dest_rel)
+          src_mode = File.stat(f).mode
+          dest_mode = File.stat(File.join(destination_root, dest_rel)).mode
+          chmod(dest_rel, src_mode) if src_mode != dest_mode
+        end
+      end
+
+      if File.exist? extra_generator_steps_file
+        eval File.read(extra_generator_steps_file), binding, extra_generator_steps_file
+      end
+    end
+
+    def valid_templates
+      valid = Dir.entries(self.class.source_root).delete_if {|e| e =~  /(^\.)|svn|CVS/ }
+    end
+
+    def load_template_config(template_dir)
+      YAML.load(File.read(File.join(template_dir, 'templates.yml'))) rescue {}
+    end
+  end
+
+end

data/lib/rubber.rb

@@ -0,0 +1,76 @@
+$:.unshift(File.dirname(__FILE__))
+
+module Rubber
+
+  @@version  = File.read(File.join(File.dirname(__FILE__), '..', 'VERSION')).chomp
+
+  def self.initialize(project_root, project_env)
+    return if defined?(RUBBER_ROOT) && defined?(RUBBER_ENV)
+
+    @@root = project_root
+    @@env = project_env
+    Object.const_set('RUBBER_ENV', project_env)
+    Object.const_set('RUBBER_ROOT', File.expand_path(project_root))
+
+    if ! defined?(Rails) && ! Rubber::Util::is_bundler?
+      # pull in basic rails env.  rubber only needs RAILS_ROOT and RAILS_ENV.
+      # We actually do NOT want the entire rails environment because it
+      # complicates bootstrap (i.e. can't run config to create db because full
+      # rails env needs db to exist as some plugin accesses model or something)
+      rails_boot_file = File.join(RUBBER_ROOT, 'config', 'boot.rb')
+      require(rails_boot_file) if File.exists? rails_boot_file
+    end
+
+    if defined?(Rails.logger) && Rails.logger
+      @@logger = Rails.logger
+    else
+      @@logger = Logger.new($stdout)
+      @@logger.level = Logger::INFO
+      @@logger.formatter = lambda {|severity, time, progname, msg| "Rubber[%s]: %s\n" % [severity, msg.to_s.lstrip]}
+    end
+
+    # conveniences for backwards compatibility with old names
+    Object.const_set('RUBBER_CONFIG', self.config)
+    Object.const_set('RUBBER_INSTANCES', self.instances)
+
+  end
+
+  def self.root
+    @@root
+  end
+
+  def self.env
+    @@env
+  end
+
+  def self.version
+    @@version
+  end
+
+  def self.logger
+    @@logger
+  end
+
+  def self.config
+    Rubber::Configuration.rubber_env
+  end
+
+  def self.instances
+    Rubber::Configuration.rubber_instances
+  end
+end
+
+
+require 'rubber/configuration'
+require 'rubber/environment'
+require 'rubber/generator'
+require 'rubber/instance'
+require 'rubber/util'
+require 'rubber/cloud'
+require 'rubber/dns'
+
+if Rubber::Util::is_rails3?
+  module Rubber
+    require 'rubber/railtie'
+  end
+end

data/lib/rubber/capistrano.rb

@@ -0,0 +1 @@
+Capistrano::Configuration.instance.load(File.join(File.dirname(__FILE__), 'recipes', 'rubber.rb'))

data/lib/rubber/cloud.rb

@@ -0,0 +1,13 @@
+require "rubber/cloud/base"
+
+module Rubber
+  module Cloud
+
+    def self.get_provider(provider, env, capistrano)
+      require "rubber/cloud/#{provider}"
+      clazz = Rubber::Cloud.const_get(provider.capitalize)
+      return clazz.new(env, capistrano)
+    end
+
+  end
+end

data/lib/rubber/cloud/aws.rb

@@ -0,0 +1,362 @@
+require 'rubygems'
+require 'AWS'
+require 'aws/s3'
+
+module Rubber
+  module Cloud
+
+    class Aws < Base
+
+      def initialize(env, capistrano)
+        super(env, capistrano)
+        @aws_env = env.cloud_providers.aws
+        @ec2 = AWS::EC2::Base.new(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+        @ec2elb = AWS::ELB::Base.new(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+        AWS::S3::Base.establish_connection!(:access_key_id => @aws_env.access_key, :secret_access_key => @aws_env.secret_access_key)
+      end
+
+      def create_instance(ami, ami_type, security_groups, availability_zone)
+        response = @ec2.run_instances(:image_id => ami, :key_name => @aws_env.key_name, :instance_type => ami_type, :security_group => security_groups, :availability_zone => availability_zone)
+        instance_id = response.instancesSet.item[0].instanceId
+        return instance_id
+      end
+
+      def create_spot_instance_request(spot_price, ami, ami_type, security_groups, availability_zone)
+        response = @ec2.request_spot_instances(:spot_price => spot_price, :image_id => ami, :key_name => @aws_env.key_name, :instance_type => ami_type, :security_group => security_groups, :availability_zone => availability_zone)
+        request_id = response.spotInstanceRequestSet.item[0].spotInstanceRequestId
+        return request_id
+      end
+
+      def describe_instances(instance_id=nil)
+        instances = []
+        opts = {}
+        opts[:instance_id] = instance_id if instance_id
+
+        response = @ec2.describe_instances(opts)
+        response.reservationSet.item.each do |ritem|
+          ritem.instancesSet.item.each do |item|
+            instance = {}
+            instance[:id] = item.instanceId
+            instance[:external_host] = item.dnsName
+            instance[:external_ip] = item.ipAddress
+            instance[:internal_host] = item.privateDnsName
+            instance[:internal_ip] = item.privateIpAddress
+            instance[:state] = item.instanceState.name
+            instance[:zone] = item.placement.availabilityZone
+            instance[:platform] = item.platform || 'linux'
+            instance[:root_device_type] = item.rootDeviceType
+            instances << instance
+          end
+        end if response.reservationSet
+
+        return instances
+      end
+
+      def destroy_instance(instance_id)
+        response = @ec2.terminate_instances(:instance_id => instance_id)
+      end
+
+      def reboot_instance(instance_id)
+        response = @ec2.reboot_instances(:instance_id => instance_id)
+      end
+
+      def stop_instance(instance_id)
+        # Don't force the stop process. I.e., allow the instance to flush its file system operations.
+        response = @ec2.stop_instances(:instance_id => instance_id, :force => false)
+      end
+
+      def start_instance(instance_id)
+        response = @ec2.start_instances(:instance_id => instance_id)
+      end
+
+      def describe_availability_zones
+        zones = []
+        response = @ec2.describe_availability_zones()
+        response.availabilityZoneInfo.item.each do |item|
+          zone = {}
+          zone[:name] = item.zoneName
+          zone[:state] =item.zoneState
+          zones << zone
+        end if response.availabilityZoneInfo
+        return zones
+      end
+
+      def create_security_group(group_name, group_description)
+        @ec2.create_security_group(:group_name => group_name, :group_description => group_description)
+      end
+
+      def describe_security_groups(group_name=nil)
+        groups = []
+
+        opts = {}
+        opts[:group_name] = group_name if group_name
+        response = @ec2.describe_security_groups(opts)
+
+        response.securityGroupInfo.item.each do |item|
+          group = {}
+          group[:name] = item.groupName
+          group[:description] = item.groupDescription
+
+          item.ipPermissions.item.each do |ip_item|
+            group[:permissions] ||= []
+            rule = {}
+
+            rule[:protocol] = ip_item.ipProtocol
+            rule[:from_port] = ip_item.fromPort
+            rule[:to_port] = ip_item.toPort
+
+            ip_item.groups.item.each do |rule_group|
+              rule[:source_groups] ||= []
+              source_group = {}
+              source_group[:account] = rule_group.userId
+              source_group[:name] = rule_group.groupName
+              rule[:source_groups] << source_group
+            end if ip_item.groups
+
+            ip_item.ipRanges.item.each do |ip_range|
+              rule[:source_ips] ||= []
+              rule[:source_ips] << ip_range.cidrIp
+            end if ip_item.ipRanges
+
+            group[:permissions] << rule
+          end if item.ipPermissions
+
+          groups << group
+          
+        end if response.securityGroupInfo
+
+        return groups
+      end
+
+      def add_security_group_rule(group_name, protocol, from_port, to_port, source)
+        opts = {:group_name => group_name}
+        if source.instance_of? Hash
+          opts = opts.merge(:source_security_group_name => source[:name], :source_security_group_owner_id => source[:account])
+        else
+          opts = opts.merge(:ip_protocol => protocol, :from_port => from_port, :to_port => to_port, :cidr_ip => source)
+        end
+        @ec2.authorize_security_group_ingress(opts)
+      end
+
+      def remove_security_group_rule(group_name, protocol, from_port, to_port, source)
+        opts = {:group_name => group_name}
+        if source.instance_of? Hash
+          opts = opts.merge(:source_security_group_name => source[:name], :source_security_group_owner_id => source[:account])
+        else
+          opts = opts.merge(:ip_protocol => protocol, :from_port => from_port, :to_port => to_port, :cidr_ip => source)
+        end
+        @ec2.revoke_security_group_ingress(opts)
+      end
+
+      def destroy_security_group(group_name)
+        @ec2.delete_security_group(:group_name => group_name)
+      end
+
+      def create_static_ip
+        response = @ec2.allocate_address()
+        return response.publicIp
+      end
+
+      def attach_static_ip(ip, instance_id)
+        response = @ec2.associate_address(:instance_id => instance_id, :public_ip => ip)
+        return response.return == "true"
+      end
+
+      def detach_static_ip(ip)
+        response = @ec2.disassociate_address(:public_ip => ip)
+        return response.return == "true"
+      end
+
+      def describe_static_ips(ip=nil)
+        ips = []
+        opts = {}
+        opts[:public_ip] = ip if ip
+        response = @ec2.describe_addresses(opts)
+        response.addressesSet.item.each do |item|
+          ip = {}
+          ip[:instance_id] = item.instanceId
+          ip[:ip] = item.publicIp
+          ips << ip
+        end if response.addressesSet
+        return ips
+      end
+
+      def destroy_static_ip(ip)
+        response = @ec2.release_address(:public_ip => ip)
+        return response.return == "true"
+      end
+
+      def create_volume(size, zone)
+        response = @ec2.create_volume(:size => size.to_s, :availability_zone => zone)
+        return response.volumeId
+      end
+
+      def attach_volume(volume_id, instance_id, device)
+        response = @ec2.attach_volume(:volume_id => volume_id, :instance_id => instance_id, :device => device)
+        return response.status
+      end
+
+      def detach_volume(volume_id)
+        @ec2.detach_volume(:volume_id => volume_id, :force => 'true')
+      end
+
+      def describe_volumes(volume_id=nil)
+        volumes = []
+        opts = {}
+        opts[:volume_id] = volume_id if volume_id
+        response = @ec2.describe_volumes(opts)
+        response.volumeSet.item.each do |item|
+          volume = {}
+          volume[:id] = item.volumeId
+          volume[:status] = item.status
+          if item.attachmentSet
+            attach = item.attachmentSet.item[0]
+            volume[:attachment_instance_id] = attach.instanceId
+            volume[:attachment_status] = attach.status
+          end
+          volumes << volume
+        end if response.volumeSet
+        return volumes
+      end
+
+      def destroy_volume(volume_id)
+        @ec2.delete_volume(:volume_id => volume_id)
+      end
+
+      def create_image(image_name)
+        ec2_key = @aws_env.key_file
+        ec2_pk = @aws_env.pk_file
+        ec2_cert = @aws_env.cert_file
+        ec2_key_dest = "/mnt/#{File.basename(ec2_key)}"
+        ec2_pk_dest = "/mnt/#{File.basename(ec2_pk)}"
+        ec2_cert_dest = "/mnt/#{File.basename(ec2_cert)}"
+
+        capistrano.put(File.read(ec2_key), ec2_key_dest)
+        capistrano.put(File.read(ec2_pk), ec2_pk_dest)
+        capistrano.put(File.read(ec2_cert), ec2_cert_dest)
+
+        arch = capistrano.capture("uname -m").strip
+        arch = case arch when /i\d86/ then "i386" else arch end
+
+        capistrano.sudo_script "create_bundle", <<-CMD
+          rvm use system
+          export RUBYLIB=/usr/lib/site_ruby/
+          unset RUBYOPT
+          nohup ec2-bundle-vol --batch -d /mnt -k #{ec2_pk_dest} -c #{ec2_cert_dest} -u #{@aws_env.account} -p #{image_name} -r #{arch} &> /tmp/ec2-bundle-vol.log &
+          echo "Creating image from instance volume..."
+          while true; do
+            if ! ps ax | grep -q "[e]c2-bundle-vol"; then exit; fi
+            echo -n .
+            sleep 1
+          done
+        CMD
+
+        capistrano.sudo_script "register_bundle", <<-CMD
+          rvm use system
+          export RUBYLIB=/usr/lib/site_ruby/
+          unset RUBYOPT
+          echo "Uploading image to S3..."
+          ec2-upload-bundle --batch -b #{@aws_env.image_bucket} -m /mnt/#{image_name}.manifest.xml -a #{@aws_env.access_key} -s #{@aws_env.secret_access_key}
+        CMD
+
+        image_location = "#{@aws_env.image_bucket}/#{image_name}.manifest.xml"
+        response = @ec2.register_image(:image_location => image_location)
+        return response.imageId
+      end
+
+      def describe_images(image_id=nil)
+        images = []
+        opts = {:owner_id => 'self'}
+        opts[:image_id] = image_id if image_id
+        response = @ec2.describe_images(opts)
+        response.imagesSet.item.each do |item|
+          image = {}
+          image[:id] = item.imageId
+          image[:location] = item.imageLocation
+          image[:root_device_type] = item.rootDeviceType
+          images << image
+        end if response.imagesSet
+        return images
+      end
+
+      def destroy_image(image_id)
+        image = describe_images(image_id).first
+        raise "Could not find image: #{image_id}, aborting destroy_image" if image.nil?
+        image_location = image[:location]
+        bucket = image_location.split('/').first
+        image_name = image_location.split('/').last.gsub(/\.manifest\.xml$/, '')
+
+        @ec2.deregister_image(:image_id => image_id)
+
+        s3_bucket = AWS::S3::Bucket.find(bucket)
+        s3_bucket.objects(:prefix => image_name).clone.each do |obj|
+          obj.delete
+        end
+        if s3_bucket.empty?
+          s3_bucket.delete
+        end
+      end
+
+      def destroy_spot_instance_request(request_id)
+        @ec2.cancel_spot_instance_requests :spot_instance_request_id => request_id
+      end
+
+      def describe_load_balancers(name=nil)
+        lbs = []
+        opts = {}
+        opts[:load_balancer_names] = name if name
+        response = @ec2elb.describe_load_balancers(opts)
+        response.describeLoadBalancersResult.member.each do |member|
+          lb = {}
+          lb[:name] = member.loadBalancerName
+          lb[:dns_name] = member.dNSName
+
+          member.availabilityZones.member.each do |zone|
+            lb[:zones] ||= []
+            lb[:zones] << zone
+          end
+
+          member.listeners.member.each do |member|
+            listener = {}
+            listener[:protocol] = member.protocol
+            listener[:port] = member.loadBalancerPort
+            listener[:instance_port] = member.instancePort
+            lb[:listeners] ||= []
+            lb[:listeners] << listener
+          end
+
+          lbs << lb
+        end if response.describeLoadBalancersResult
+        return lbs
+      end
+
+      def describe_spot_instance_requests(request_id=nil)
+        requests = []
+        opts = {}
+        opts[:spot_instance_request_id] = request_id if request_id
+        response = @ec2.describe_spot_instance_requests(opts)
+        response.spotInstanceRequestSet.item.each do |item|
+          request = {}
+          request[:id] = item.spotInstanceRequestId
+          request[:spot_price] = item.spotPrice
+          request[:state] = item.state
+          request[:created_at] = item.createTime
+          request[:type] = item.launchSpecification.instanceType
+          request[:image_id] = item.launchSpecification.imageId
+          request[:instance_id] = item.instanceId
+          requests << request
+        end if response.spotInstanceRequestSet
+        return requests
+      end
+
+      # resource_id is any Amazon resource ID (e.g., instance ID or volume ID)
+      # tags is a hash of tag_name => tag_value pairs
+      def create_tags(resource_id, tags)
+        # Tags needs to be an array of hashes, not one big hash, so break it down.
+        @ec2.create_tags(:resource_id => resource_id, :tag => tags.collect { |k, v| { k.to_s => v.to_s } })
+      end
+
+    end
+
+  end
+end