auser-poolparty 0.1.2 → 0.2.2

data/lib/poolparty/remote_instance.rb

@@ -1,250 +0,0 @@
-module PoolParty
-  class RemoteInstance
-    # ############################
-    include Remoter
-    # ############################
-    include PoolParty
-    include Callbacks
-    include FileWriter
-    
-    attr_reader :ip, :instance_id, :name, :status, :launching_time, :stack_installed, :keypair 
-    attr_accessor :name, :number, :scp_configure_file, :configure_file, :plugin_string, :keypair
-    
-    # CALLBACKS
-    after :install, :mark_installed
-    after :configure, :associate_public_ip
-    def initialize(obj={})
-      super
-      
-      @ip = obj[:ip]
-      @instance_id = obj[:instance_id]      
-      @name = obj[:name] || "node"
-      @number = obj[:number] || 0 # Defaults to the master
-      @status = obj[:status] || "running"
-      @launching_time = obj[:launching_time] || Time.now
-      @keypair = obj[:keypair] || Application.keypair rescue ""
-    end
-    
-    # Host entry for this instance
-    def hosts_entry
-      "#{@ip} #{name}"
-    end
-    # Internal host entry for this instance
-    def local_hosts_entry
-      "127.0.0.1 #{name}\n127.0.0.1 localhost.localdomain localhost ubuntu"
-    end
-    # Node entry for heartbeat
-    def node_entry
-      "node #{name}"
-    end    
-    # Internal naming scheme
-    def name
-      "#{@name}#{@number}"
-    end
-    # Entry for the heartbeat config file
-    def heartbeat_entry
-      "#{name} #{ip} #{Application.managed_services}"
-    end
-    # Entry for haproxy
-    def haproxy_entry
-      "\tserver #{name} #{@ip}:#{Application.client_port} weight 1 minconn 3 maxconn 6 check inter 20000 check"
-    end
-    def haproxy_resources_entry
-      "#{name} #{@ip}"
-    end
-    # Is this the master?
-    def master?
-      @number == 0
-    end
-    def secondary?
-      @number == 1
-    end
-    def set_hosts(c)
-    end
-    # Let's define some stuff for monit
-    %w(stop start restart).each do |cmd|
-      define_method "#{cmd}_with_monit" do
-        ssh("monit #{cmd} all")
-      end
-    end
-    def configure_tasks(quiet=true)
-      {
-        :setup_pems => setup_pems,
-        :move_hostfile => change_hostname,
-        :config_master => configure_master,
-        :move_config_file => move_config_file,
-        :mount_s3_drive => mount_s3_drive,
-        :update_plugins => update_plugin_string,        
-        :configure_authkeys => configure_authkeys,
-        :configure_resource_d => configure_resource_d,
-        :configure_haproxy => setup_haproxy,
-        :configure_heartbeat => configure_heartbeat,
-        :user_tasks => user_tasks,
-        :node_list => move_node_list
-      }#.map {|k,v| {k.to_sym => v.nice_runnable(quiet)} }.inject({}) {|a,s| s.merge(a) }
-    end
-    def user_tasks
-      @@user_tasks ||= []
-    end
-    def move_config_file
-      <<-EOC
-        #{if_exists "config.yml", "mv #{remote_base_tmp_dir}/config.yml ~/.config"}
-        mkdir -p ~/.ec2
-        #{if_exists "keypair", "mv #{remote_base_tmp_dir}/keypair ~/.ec2/#{Application.keypair_name}"}        
-      EOC
-    end
-    def configure_heartbeat      
-      cmd=<<-EOC
-        mv #{remote_base_tmp_dir}/ha.cf /etc/ha.d/ha.cf
-        /etc/init.d/heartbeat start
-      EOC
-      Master.requires_heartbeat? ? cmd : ""
-    end    
-    def configure_authkeys
-      <<-EOC
-        mkdir -p /etc/ha.d
-        #{if_exists "authkeys", "mv #{remote_base_tmp_dir}/authkeys /etc/ha.d/"}
-      EOC
-    end
-    
-    def configure_master
-      if master?
-        <<-EOC
-          pool maintain -c ~/.config -l ~/plugins
-        EOC
-      else
-        ""
-      end
-    end
-    
-    def configure_resource_d
-      <<-EOC
-        mkdir -p /etc/ha.d/resource.d/
-        #{if_exists "cloud_master_takeover", "mv #{remote_base_tmp_dir}/cloud_master_takeover /etc/ha.d/resource.d"}
-        #{if_dir_exists "resource.d/", "mv #{remote_base_tmp_dir}/resource.d/* /etc/ha.d/resource.d"}
-      EOC
-    end
-    
-    # def configure_monit
-    #   <<-EOC
-    #     mv #{remote_base_tmp_dir}/monitrc /etc/monit/monitrc
-    #     mkdir -p /etc/monit.d/
-    #     mv #{remote_base_tmp_dir}/monit.d/* /etc/monit.d/
-    #     chown #{Application.username} /etc/monit/monitrc
-    #     chmod 700 /etc/monit/monitrc
-    #   EOC
-    # end
-    
-    def setup_pems
-      <<-EOC
-        #{if_exists "cert.pem", "mv #{remote_base_tmp_dir}/cert-*.pem #{Application.keypair_path}/cert-CLOUD.pem"}
-        #{if_exists "pk.pem", "mv #{remote_base_tmp_dir}/pk-*.pem #{Application.keypair_path}/pk-CLOUD.pem"}
-      EOC
-    end
-    
-    def change_hostname
-      <<-EOC
-        #{if_exists "#{name}-hosts", "mv #{remote_base_tmp_dir}/#{name}-hosts /etc/hosts"}
-        hostname -v #{name}
-      EOC
-    end
-    
-    def setup_haproxy
-      <<-EOS
-        mv #{remote_base_tmp_dir}/haproxy /etc/haproxy.cfg
-        sed -i "s/ENABLED=0/ENABLED=1/g" /etc/default/haproxy
-        sed -i 's/SYSLOGD=""/SYSLOGD="-r"/g' /etc/default/syslogd
-        echo "local0.* /var/log/haproxy.log" >> /etc/syslog.conf && /etc/init.d/sysklogd restart
-        /etc/init.d/haproxy restart
-      EOS
-    end
-    
-    def mount_s3_drive
-      if Application.shared_bucket.empty?
-        ""
-      else
-        <<-EOC
-          mkdir -p /data && /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
-        EOC
-      end
-    end
-    def move_node_list
-      <<-EOC
-        #{if_exists node_list_name, "mv #{remote_base_tmp_dir}/nodes.lst ~/.#{node_list_name}"}
-      EOC
-    end
-    def node_list_name
-      self.class.node_list_name
-    end
-    def self.node_list_name
-      "nodes.lst"
-    end
-    # Installs with one commandline and an scp, rather than 10
-    def install
-    end
-    # Login to store the authenticity
-    def login_once
-      run_now "ls -l"
-    end
-    # Associate a public ip if it is set and this is the master
-    def associate_public_ip(c)
-      associate_address_with(Application.public_ip, @instance_id) if master? && Application.public_ip && !Application.public_ip.empty?
-    end
-    # Become the new master
-    def become_master
-      @master = Master.new
-      @number = 0
-      @master.nodes[0] = self
-      @master.configure_cloud
-      configure
-    end
-    # Placeholder
-    def configure      
-    end
-    def update_plugin_string
-      dir = File.basename(Application.plugin_dir)
-      
-      if_exists "plugins.tar.gz", "mkdir -p #{dir} && tar -zxf #{remote_base_tmp_dir}/plugins.tar.gz -C #{dir}"
-    end
-    def if_exists(file, dothis)
-      "if [ -f #{remote_base_tmp_dir}/#{file} ]; then #{dothis}; fi"
-    end
-    def if_dir_exists(dir, dothis)
-      "if [ -d #{remote_base_tmp_dir}/#{dir} ]; then #{dothis}; fi"
-    end
-    # Is this the master and if not, is the master running?
-    def is_not_master_and_master_is_not_running?
-      !master? && !Master.is_master_responding?
-    end
-    # User conf file if it exists, or default one
-    def conf_file(name)
-      user_conf = File.join(PoolParty.user_dir, "config", name)
-      if File.file?(user_conf)
-        File.join(PoolParty.user_dir, "config", name)
-      else
-        File.join(PoolParty.root_dir, "config", name)
-      end        
-    end
-    # Description in the rake task
-    def description
-      case @status
-      when "running"
-        "#{@number}: INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
-      when "shutting-down"
-        "(terminating) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
-      when "pending"
-        "(booting) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
-      end
-    end
-    def stack_installed?
-      @stack_installed ||= run("cat ~/.installed")
-    end
-    def mark_installed(caller=nil)
-      run_now("echo 'installed' > ~/.installed")
-      @stack_installed = true
-    end
-    def base_install_script
-      "#{root_dir}/config/installers/#{Application.os.downcase}_install.sh"
-    end
-  end
-end

data/lib/poolparty/remoter.rb

@@ -1,171 +0,0 @@
-=begin rdoc
-  Handle the remoting aspects of the remote_instances
-=end
-require 'open4'
-module PoolParty
-  module Remoter        
-    module ClassMethods
-      def ssh_string
-        (["ssh"] << ssh_array).join(" ")
-      end
-      def ssh_array
-        ["-o StrictHostKeyChecking=no", "-l '#{Application.username}'", "-i '#{Application.keypair_path}'"]
-      end
-      def rsync_string
-        "rsync --delete -azP -e '#{ssh_string}' "
-      end
-    end
-    
-    module InstanceMethods      
-      include Callbacks
-      include Scheduler
-      
-      def rsync_tasks local, remote
-        reset!
-        returning scp_tasks do |tasks|
-          target_hosts.each do |ip|
-            tasks << "#{self.class.rsync_string} #{local} #{ip}:#{remote}"
-          end
-        end
-      end
-      
-      def remote_command_tasks commands
-        reset!
-        commands = commands.join ' && ' if commands.is_a? Array
-        
-        returning ssh_tasks do |tasks|
-          target_hosts.each do |ip|
-            ssh_tasks << "#{self.class.ssh_string} #{ip} '#{commands}'"
-          end
-        end
-      end
-      
-      def scp local, remote, opts={}          
-        cmd = self.class.rsync_string
-        arr = []
-        
-        target_hosts.each do |ip|
-          arr << "#{cmd} #{local} #{ip}:#{remote}"
-        end          
-        
-        run_array_of_tasks(arr)
-      end
-
-      def single_scp local, remote, opts={}
-        scp local, remote, opts.merge({:single => self.ip})
-      end
-
-      def ssh command="", opts={}, &block
-        cmd = self.class.ssh_string
-        arr = []
-        
-        if command.empty?
-          system("#{cmd} #{self.ip}") if self.class == RemoteInstance
-        else
-          target_hosts.each do |ip|
-            arr << "#{cmd} #{ip} '#{command.runnable}'"
-          end
-        end
-                
-        run_array_of_tasks arr
-      end
-      
-      def run_now command        
-        run command unless command.empty?
-      end
-            
-      def ssh_tasks;@ssh_tasks ||= [];end
-      def scp_tasks;@scp_tasks ||= [];end
-            
-      def reset!
-        @ssh_tasks = @scp_tasks = nil
-        @hosts = nil
-      end
-            
-      def execute_tasks(opts={})
-        # reset!
-        
-        set_hosts(nil) unless opts[:dont_set_hosts]
-        
-        yield if block_given?
-        
-        run_array_of_tasks(scp_tasks)
-        run_array_of_tasks(ssh_tasks)
-      end
-      
-      def target_hosts
-        @hosts ||= Master.collect_nodes {|a| a.ip }
-      end
-      
-      def run_array_of_tasks(task_list)
-        unless task_list.size == 0
-          task_list.each do |task|
-            add_task {Kernel.system("#{task}")}
-          end
-          run_thread_list
-        end
-      end
-      
-      def set_hosts(c=nil)
-      end
-      
-      # Nearly Directly from vlad
-      def run command, on=self
-        cmd = [self.class.ssh_string, on.ip].compact
-        result = []
-
-        commander = cmd.join(" ") << " \"#{command.runnable}\""        
-        
-        pid, inn, out, err = Open4::popen4(commander)
-
-        inn.sync   = true
-        streams    = [out, err]
-        out_stream = {
-          out => $stdout,
-          err => $stderr,
-        }
-
-        # Handle process termination ourselves
-        status = nil
-        Thread.start do
-          status = Process.waitpid2(pid).last
-        end
-
-        until streams.empty? do
-          # don't busy loop
-          selected, = select streams, nil, nil, 0.1
-
-          next if selected.nil? or selected.empty?
-
-          selected.each do |stream|
-            if stream.eof? then
-              streams.delete stream if status # we've quit, so no more writing
-              next
-            end
-
-            data = stream.readpartial(1024)
-            out_stream[stream].write data
-
-            if stream == err and data =~ /^Password:/ then
-              inn.puts sudo_password
-              data << "\n"
-              $stderr.write "\n"
-            end
-
-            result << data
-          end
-        end
-
-        PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
-
-        result.join
-      end
-      
-    end
-  
-    def self.included(receiver)
-      receiver.extend         ClassMethods
-      receiver.send :include, InstanceMethods
-    end
-  end
-end

data/lib/poolparty/remoting.rb

@@ -1,137 +0,0 @@
-module PoolParty
-  extend self
-  
-  class Remoting
-    include PoolParty
-    include Ec2Wrapper
-    include Scheduler
-    
-    # == GENERAL METHODS    
-    # == LISTING
-    # List all the running instances associated with this account
-    def list_of_running_instances(list = list_of_nonterminated_instances)
-      list.select {|a| a[:status] =~ /running/}
-    end
-    # Get a list of the pending instances
-    def list_of_pending_instances(list = list_of_nonterminated_instances)
-      list.select {|a| a[:status] =~ /pending/}
-    end
-    # list of shutting down instances
-    def list_of_terminating_instances(list = list_of_nonterminated_instances)
-      list.select {|a| a[:status] =~ /shutting/}
-    end
-    # list all the nonterminated instances
-    def list_of_nonterminated_instances(list = list_of_instances)
-      list.reject {|a| a[:status] =~ /terminated/}
-    end
-    # List the instances for the current key pair, regardless of their states
-    def list_of_instances(keypair = Application.keypair)
-      get_instances_description.select {|a| a[:keypair] == keypair}
-    end
-    # List all instances, regardless of their key pairs or states
-    def list_of_all_instances
-      a_list = []
-      cloud_keypairs.each {|keypair| list_of_instances(keypair).each { |inst| a_list << inst } }
-      a_list
-    end
-    # list of keypairs for the current AWS access key and secret key
-    def cloud_keypairs
-      get_instances_description.each {|a| a[:keypair] = "no-keypair" unless a[:keypair] } # Get rid of the instances not launched with a keypair
-      instances = get_instances_description.sort{|x,y| x[:keypair] <=> y[:keypair]}
-      keypair = nil
-      instances.map {|a| keypair != a[:keypair] ? (keypair = a[:keypair]; keypair) : nil }.compact
-    end
-    # Get number of pending instances
-    def number_of_pending_instances(list = list_of_pending_instances)
-      list.size
-    end
-    # get the number of running instances
-    def number_of_running_instances(list = list_of_running_instances)
-      list.size
-    end
-    # get the number of pending and running instances for a list
-    def number_of_pending_and_running_instances_for_list(a_list = list_of_instances)
-      running_list = list_of_running_instances( a_list )
-      pending_list = list_of_pending_instances( a_list )
-      number_of_running_instances(running_list) + number_of_pending_instances(pending_list)
-    end
-    # get the number of pending and running instances for a keypair
-    def number_of_pending_and_running_instances(keypair = Application.keypair)
-      a_list = list_of_nonterminated_instances( list_of_instances(keypair) )
-      number_of_pending_and_running_instances_for_list( a_list )
-    end
-    # get the number of pending and running instances
-    def number_of_all_pending_and_running_instances
-      a_list = list_of_nonterminated_instances( list_of_all_instances )
-      number_of_pending_and_running_instances_for_list( a_list )
-    end
-    # == LAUNCHING
-    # Request to launch a new instance
-    def request_launch_new_instance
-      if can_start_a_new_instance?
-        request_launch_one_instance_at_a_time
-        return true
-      else
-        return false
-      end
-    end
-    # Can we start a new instance?
-    def can_start_a_new_instance?
-      maximum_number_of_instances_are_not_running?
-    end
-    # Are the maximum number of instances running?
-    def maximum_number_of_instances_are_not_running?
-      list_of_running_instances.size < Application.maximum_instances
-    end
-    # Request to launch a number of instances
-    def request_launch_new_instances(num=1)
-      out = []
-      num.times {out << request_launch_one_instance_at_a_time}
-      out
-    end
-    # Launch one instance at a time
-    def request_launch_one_instance_at_a_time
-      reset!
-      while !number_of_pending_instances.zero?
-        wait "5.seconds"
-        reset!
-      end
-      return launch_new_instance!
-    end
-    # == SHUTDOWN
-    # Terminate all running instances
-    def request_termination_of_running_instances
-      list_of_running_instances.each {|a| terminate_instance!(a[:instance_id])}
-    end
-    # Request termination of all instances regardless of their state (includes pending instances)
-    def request_termination_of_all_instances
-      list_of_instances.each {|a| terminate_instance!(a[:instance_id])}
-    end
-    # Terminate instance by id
-    def request_termination_of_instance(id)
-      if can_shutdown_an_instance?
-        terminate_instance! id
-        return true
-      else
-        return false
-      end
-    end
-    # Can we shutdown an instance?
-    def can_shutdown_an_instance?
-      minimum_number_of_instances_are_running?
-    end
-    # Are the minimum number of instances running?
-    def minimum_number_of_instances_are_running?
-      list_of_running_instances.size > Application.minimum_instances
-    end
-    # Get the cached running_instances
-    def running_instances
-      @running_instances ||= update_instance_values
-    end
-    # Update the instance values
-    def update_instance_values
-      @running_instances = list_of_running_instances.collect {|a| RemoteInstance.new(a) }.sort
-    end
-  end
-    
-end