pvcglue 0.1.39 → 0.9.0

data/lib/pvcglue/configuration.rb

@@ -38,7 +38,10 @@ module Pvcglue
           init(:local_cloud_manager)
           @cloud_manager = @local_cloud_manager
         else
-          init(:cloud_manager) || configure_manager
+          unless init(:cloud_manager)
+            say('The manager has not been configured.')
+            configure_manager
+          end
         end
 
         # raise(Thor::Error, "The manager has not been configured.  :(") if cloud_manager.nil?
@@ -57,7 +60,6 @@ module Pvcglue
       end
 
       def configure_manager
-        say('The manager has not been configured.')
         manager = ask('What is the IP address or host name of the manager?')
         default = !no?('Will this be the default manager? (Y/n)')
         file_name = default ? user_file_name : project_file_name
@@ -188,14 +190,50 @@ module Pvcglue
         end
       end
 
-      def web_app_base_dir
-        '/sites'
+      # def web_app_base_dir
+      #   # '/sites'
+      #   '~/www'
+      #   # "/home/#{user_name}/.ssh"
+      # end
+
+      def build_log_extra_dir
+        @build_log_extra_dir ||= begin
+          option = Pvcglue.command_line_options[:save_before_upload]
+          if option != 'save_before_upload'
+            dir_name = option
+          else
+            dir_name = Time.now.strftime('%Y-%m-%d-%H%M%S')
+          end
+          result = File.join(pvcglue_tmp_dir, dir_name)
+          `mkdir -p '#{result}'`
+          raise $?.inspect unless $?.exitstatus == 0
+          result
+        end
+      end
+
+      def build_log_extra_filename(minion, user, remote_filename)
+        # local_filename = File.basename(remote_filename)
+        relative = remote_filename.sub(Pvcglue.cloud.web_app_base_dir, '/project')
+        local_filename = relative.sub(/\A\//, '').gsub(/\//, '__')
+        # local_filename = File.basename(remote_filename)
+        versioned_filename(File.join(build_log_extra_dir, "#{minion.machine_name}--#{user}--#{local_filename}"))
+      end
+
+      # TODO:  Refactor to a utilities module or something
+      def versioned_filename(base, first_suffix='.00')
+        suffix = nil
+        filename = base
+        while File.exists?(filename)
+          suffix = (suffix ? suffix.succ : first_suffix)
+          filename = base + suffix
+        end
+        return filename
       end
     end
 
   end
 
-  # --------------------------------------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------------------------------------
 
   def self.configuration
     @configuration ||= Configuration.new

data/lib/pvcglue/connection.rb

@@ -0,0 +1,236 @@
+module Pvcglue
+  # TODO:  Check ssh config
+  ## https://puppet.com/blog/speed-up-ssh-by-reusing-connections
+  # ~/.ssh/config
+  #Host *
+  #ControlMaster auto
+  ##ControlPath ~/.ssh/sockets/%r@%h-%p
+  #ControlPath ~/.ssh/%r@%h-%p
+  #ControlPersist 600
+  #
+  # Way Faster!!!
+  #
+
+  class Connection
+    def initialize(minion)
+      @minion = minion
+    end
+
+    attr_accessor :minion
+    attr_accessor :minion_state_data
+
+    def file_exists?(user, file)
+      result = ssh?(user, '', "test -e #{file}")
+      result.exitstatus == 0
+    end
+
+    def mkdir_p(user, remote_dir, owner = nil, group = nil, permissions = nil)
+      ssh!(user, '', "mkdir -p #{remote_dir}")
+      chown_chmod(user, remote_dir, owner, group, permissions)
+    end
+
+    def ssh_retry_wait(user, options, cmd, times, wait)
+      tries = 0
+      begin
+        result = ssh?(user, options, cmd)
+        unless result.exitstatus == 0
+          Pvcglue.logger.info("Command `#{cmd}` failed, retrying...")
+          sleep(wait)
+        end
+        tries += 1
+        raise "Exceeded #{times} retries:  #{result.inspect}" if tries >= times
+      end until result.exitstatus == 0
+    end
+
+    def escape_ssh_command(cmd)
+      # cmd.gsub(/'/, "\\'")
+      # cmd.gsub(/'/) { |match| "\\#{match}" }
+      # cmd.gsub(/'/) { |match| '\\' + match } # "CREATEDB PASSWORD 'monkey'" #=> "CREATEDB PASSWORD \'monkey\'"
+      # if cmd.include?("'") || cmd.include?('"')
+      # # if cmd.include?('"')
+      #   # "<<'EOF'\n#{cmd}\nEOF"
+      #   # cmd.gsub(/'/) { |match| %Q('"'"') }
+      #   # "$'" + cmd.gsub(/'/) { |match| '\\' + match } + "'"
+      #   '"' + cmd.gsub(/"/) { |match| '\\' + match } + '"'
+      # else
+      #   "'#{cmd}'"
+      # end
+      # Another possible solution:  http://stackoverflow.com/a/21761956/444774
+      '"' + cmd.gsub(/"/) { |match| '\\' + match } + '"'
+    end
+
+    def ssh?(user, options, cmd)
+      # if cmd.include?("'") # DONE:  Quick fix, should be refactored to do proper escaping
+      #   system_command?(%Q(ssh #{user}@#{minion.public_ip} #{options} "#{cmd}"))
+      # else
+      #   system_command?(%Q(ssh #{user}@#{minion.public_ip} #{options} '#{cmd}'))
+      # end
+      system_command?(%Q(ssh #{user}@#{minion.public_ip} #{options} #{escape_ssh_command(cmd)}))
+    end
+
+    def ssh!(user, options, cmd)
+      result = ssh?(user, options, cmd)
+      raise result.inspect unless result.exitstatus == 0
+    end
+
+    def system_command?(cmd)
+      Pvcglue.logger.debug { cmd }
+      system(cmd)
+      Pvcglue.logger.debug { "exit_code=#{$?.exitstatus}" }
+      $?
+    end
+
+    def system_command!(cmd)
+      result = system_command?(cmd)
+      raise result.inspect unless result.exitstatus == 0
+    end
+
+    def run_get_stdout(user, options, cmd)
+      full_cmd = "ssh #{user}@#{minion.public_ip} #{options} #{escape_ssh_command(cmd)}"
+      Pvcglue.logger.debug { full_cmd }
+      result = `#{full_cmd}`
+      Pvcglue.verbose? { result }
+      Pvcglue.logger.debug { "exit_code=#{$?.to_i}" }
+      result
+    end
+
+    def run_get_stdout!(user, options, cmd)
+      result = run_get_stdout(user, options, cmd)
+      raise $?.inspect unless $?.exitstatus == 0
+      result
+    end
+
+    def run!(user, options, cmd)
+      ssh!(user, options, cmd)
+    end
+
+    def run?(user, options, cmd)
+      ssh?(user, options, cmd)
+      # puts user.inspect
+      # cmd = 'pwd'
+      # full_command = "ssh root@#{minion.public_ip} '#{cmd}'"
+      # full_command = "ssh root@#{minion.public_ip} 'ls -ahl'"
+      # puts "running:  #{full_command}"
+      # puts `#{full_command}`
+      # 1.times do
+      # 100.times do
+      # puts "running:  #{full_command}"
+      # puts `#{full_command}`
+      # puts `ls -ahl ~/.ssh/config`
+      # system %Q(ssh root@#{minion.public_ip} -o strictHostKeyChecking=no -t 'pwd')
+      # ap system %Q(ssh root@#{minion.public_ip} 'test -e test')
+      # ap system %Q(ssh root@#{minion.public_ip} -t 'test -e test')
+      # ap $?
+      # ap system %Q(ssh root@#{minion.public_ip} 'test -e .bashrc')
+      # ap $?
+      # ap file_exists?(:root, 'test')
+      # ap file_exists?(:root, '.bashrc')
+      # end
+    end
+
+    def read_from_file(user, file)
+      tmp_file = Tempfile.new('pvc')
+      begin
+        download_file(user, file, tmp_file.path)
+        data = tmp_file.read
+        Pvcglue.verbose? { data }
+      ensure
+        tmp_file.close
+        tmp_file.unlink # deletes the temp file
+      end
+      data
+    end
+
+    def read_from_file_if_exists?(user, file)
+      tmp_file = Tempfile.new('pvc')
+      begin
+        result = download_file(user, file, tmp_file.path, false)
+        if result.exitstatus == 0
+          data = tmp_file.read
+        elsif result.exitstatus == 1
+          data = nil
+        else
+          raise $?.inspect
+        end
+        Pvcglue.verbose? { data }
+      ensure
+        tmp_file.close
+        tmp_file.unlink # deletes the temp file
+      end
+      data
+
+    end
+
+    def write_to_file_from_template(user, template_file_name, file, owner = nil, group = nil, permissions = nil)
+      Pvcglue.logger.debug { "Writing to #{file} from template '#{template_file_name}'" }
+      template = Tilt.new(Pvcglue.template_file_name(template_file_name))
+      data = template.render(self, minion: minion)
+
+      write_to_file(user, data, file, owner, group, permissions)
+    end
+
+    def write_to_file(user, data, file, owner = nil, group = nil, permissions = nil)
+      tmp_file = Tempfile.new('pvc')
+      begin
+        tmp_file.write(data)
+        tmp_file.flush
+        upload_file(user, tmp_file.path, file, owner, group, permissions)
+        Pvcglue.verbose? { data }
+      ensure
+        tmp_file.close
+        tmp_file.unlink # deletes the temp file
+      end
+    end
+
+    def download_file(user, remote_file, local_file, raise_error = true)
+      cmd = %{scp #{user}@#{minion.public_ip}:#{remote_file} #{local_file}}
+      if raise_error
+        system_command!(cmd)
+      else
+        system_command?(cmd)
+      end
+    end
+
+    def upload_file(user, local_file, remote_file, owner = nil, group = nil, permissions = nil)
+      if Pvcglue.command_line_options[:save_before_upload] && file_exists?(user, remote_file)
+        download_file(user, remote_file, Pvcglue.configuration.build_log_extra_filename(minion, user, remote_file))
+      end
+      system_command!(%{scp #{local_file} #{user}@#{minion.public_ip}:#{remote_file}})
+      chown_chmod(user, remote_file, owner, group, permissions)
+    end
+
+    def chown_chmod(user, remote, owner, group, permissions = nil)
+      unless owner.nil? && group.nil?
+        raise('Invalid owner or group for chown') if owner.nil? || group.nil?
+        chown(user, remote, owner, group)
+      end
+      unless permissions.nil?
+        chmod(user, remote, permissions)
+      end
+    end
+
+    def chown(user, remote, owner, group, options = nil)
+      ssh!(user, '', "chown #{options} #{owner}:#{group} #{remote}")
+    end
+
+    def chmod(user, remote, permissions, options = nil)
+      ssh!(user, '', "chmod #{options} #{permissions} #{remote}")
+    end
+
+    def file_matches?(user, data, remote_file)
+      # NOTE:  This could be optimized
+      return false unless file_exists?(user, remote_file)
+      read_from_file(user, remote_file) == data
+    end
+
+    def rsync_up (user, options, local_source_dir, remote_destination_dir, mkdir = true)
+      mkdir_p(user, remote_destination_dir) if mkdir
+      cmd = ''
+      # cmd += "mkdir -p #{remote_destination_dir} && "
+      cmd += %(rsync #{options} #{local_source_dir}/ #{user}@#{minion.public_ip}:#{remote_destination_dir}/)
+      # cmd = (%(rsync -rzv --exclude=maintenance.on --delete -e 'ssh -p #{Pvcglue.cloud.port_in_node_context}' #{source_dir}/ #{node.get(:user)}@#{node.host}:#{dest_dir}/))
+      system_command!(cmd)
+    end
+
+  end
+end

data/lib/pvcglue/custom_hashie.rb

@@ -0,0 +1,3 @@
+class SafeMash < Hashie::Mash
+  include Hashie::Extensions::Mash::SafeAssignment
+end

data/lib/pvcglue/db.rb

@@ -96,8 +96,8 @@ module Pvcglue
     end
 
     def self.db_host_public
-      node = Pvcglue.cloud.find_node('db')
-      node['db']['public_ip']
+      # Assume 1 pg server
+      Pvcglue.cloud.minions_filtered('pg').values.first.public_ip
     end
 
 
@@ -128,12 +128,15 @@ module Pvcglue
 
 
       def pg_dump(db, file_name, fast)
+        # TODO:  Refactor into the db package
         file_name = self.class.file_helper(file_name)
 
         if db.kind == :remote
-          host = Pvcglue.cloud.nodes_in_stage('db')['db']['public_ip']
+          minion = Pvcglue.cloud.minions_filtered('pg').values.first
+          # Assume 1 pg server
+          host = minion.public_ip
           port = Pvcglue.cloud.port_in_context(:shell)
-          user = 'deploy'
+          user = minion.remote_user_name
         end
 
         cmd = "pg_dump -Fc --no-acl --no-owner -h #{db.host} -p #{db.port}"
@@ -153,43 +156,46 @@ module Pvcglue
           unless Pvcglue.run_remote(host, port, user, cmd)
             puts "ERROR:"
             puts $?.inspect
-            raise(Thor::Error, "Error:  #{$?}")
+            raise("Error:  #{$?}")
           end
 
           cmd = %{scp -P #{port} #{user}@#{host}:#{file_name} #{file_name}}
           puts "Running `#{cmd}`"
 
           unless system cmd
-            raise(Thor::Error, "Error:  #{$?}")
+            raise("Error:  #{$?}")
           end
         else
           unless system(cmd)
             puts "ERROR:"
             puts $?.inspect
-            raise(Thor::Error, "Error:  #{$?}")
+            raise("Error:  #{$?}")
           end
         end
       end
 
       def pg_restore(db, file_name, fast = false)
+        # TODO:  Refactor into the db package
         Pvcglue.cloud.stage_name == 'production' && destroy_prod?(db)
         file_name = self.class.file_helper(file_name)
 
+
         if db.kind == :remote
-          host = Pvcglue.cloud.nodes_in_stage('db')['db']['public_ip']
+          minion = Pvcglue.cloud.minions_filtered('pg').values.first
+          host = minion.public_ip
           port = Pvcglue.cloud.port_in_context(:shell)
-          user = 'deploy'
+          user = minion.remote_user_name
 
           # cmd = %{scp -P #{port} #{file_name} #{user}@#{host}:#{file_name}}
           cmd = %{rsync -avhPe "ssh -p #{port}" --progress #{file_name} #{user}@#{host}:#{file_name}}
           unless system cmd
-            raise(Thor::Error, "Error:  #{$?}")
+            raise("Error:  #{$?}")
           end
 
           unless fast
             # Drop and recreate DB
-            Pvcglue::Packages.apply('postgresql-app-stage-db-drop'.to_sym, :build, Pvcglue.cloud.nodes_in_stage('db'))
-            Pvcglue::Packages.apply('postgresql-app-stage-conf'.to_sym, :build, Pvcglue.cloud.nodes_in_stage('db'))
+            Pvcglue::Packages.apply('postgresql-app-stage-db-drop'.to_sym, :build, Pvcglue.cloud.minions_filtered('db'))
+            Pvcglue::Packages.apply('postgresql-app-stage-conf'.to_sym, :build, Pvcglue.cloud.minions_filtered('db'))
           end
         end
 
@@ -205,7 +211,7 @@ module Pvcglue
           end
         else
           unless system(" PGPASSWORD=#{db.password} #{cmd}")
-            raise(Thor::Error, "Error:  #{$?}")
+            raise("Error:  #{$?}")
           end
         end
 

data/lib/pvcglue/digital_ocean.rb

@@ -0,0 +1,21 @@
+module Pvcglue
+  class DigitalOcean
+
+    def self.client
+      @@client ||= get_client
+    end
+
+    def self.get_client
+      access_token = YAML::load(File.open(File.join(ENV['HOME'], '.config/doctl/config.yaml')))['access-token']
+      ::DropletKit::Client.new(access_token: access_token)
+    end
+
+    def self.get_ip_addresses(droplet)
+      ips = ::SafeMash.new
+      droplet.networks.v4.each do |network|
+        ips[network.type] = network.ip_address
+      end
+      ips
+    end
+  end
+end

data/lib/pvcglue/env.rb

@@ -6,16 +6,18 @@ module Pvcglue
     desc "push", "push"
 
     def push
-      Pvcglue::Packages.apply('env-push'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
-      self.class.clear_stage_env_cache
-      Pvcglue::Packages.apply('app-env-file'.to_sym, :env, Pvcglue.cloud.nodes_in_stage('web'))
+      raise('Not implemented')
+      # Pvcglue::Packages.apply('env-push'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
+      # self.class.clear_stage_env_cache
+      # Pvcglue::Packages.apply('app-env-file'.to_sym, :env, Pvcglue.cloud.nodes_in_stage('web'))
     end
 
     desc "pull", "pull"
 
     def pull
-      Pvcglue::Packages.apply('env-pull'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
-      self.class.clear_stage_env_cache
+      raise('Not implemented')
+      # Pvcglue::Packages.apply('env-pull'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
+      # self.class.clear_stage_env_cache
     end
 
     desc "list", "list"
@@ -38,10 +40,12 @@ module Pvcglue
 
     def set(*args)
       self.class.initialize_stage_env
+      Pvcglue.logger.debug { args.each { |arg| arg.inspect } }
       options = Hash[args.each.map { |l| l.chomp.split('=') }]
+      Pvcglue.logger.debug { options.inspect }
       Pvcglue.cloud.stage_env.merge!(options)
       self.class.save_stage_env
-      Pvcglue::Packages.apply('app-env-file'.to_sym, :env, Pvcglue.cloud.nodes_in_stage('web'))
+      self.class.apply_changes
     end
 
     desc "unset", "remove environment variable(s) for the stage XYZ [ZZZ]"
@@ -50,7 +54,7 @@ module Pvcglue
       self.class.initialize_stage_env
       args.each { |arg| puts "WARNING:  Key '#{arg}' not found." unless Pvcglue.cloud.stage_env.delete(arg) }
       self.class.save_stage_env
-      Pvcglue::Packages.apply('app-env-file'.to_sym, :env, Pvcglue.cloud.nodes_in_stage('web'))
+      self.class.apply_changes
     end
 
     desc "rm", "alternative to unset"
@@ -63,10 +67,11 @@ module Pvcglue
     # ------------------------------------------------------------------------------------------------------------------
 
     def self.initialize_stage_env
-      unless read_cached_stage_env
-        Pvcglue::Packages.apply('env-get-stage'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
-        write_stage_env_cache
-      end
+      Pvcglue::Packages::Secrets.load_for_stage
+      # unless read_cached_stage_env
+      #   Pvcglue::Packages.apply('env-get-stage'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
+      #   write_stage_env_cache
+      # end
       merged = stage_env_defaults.merge(Pvcglue.cloud.stage_env)
       if merged != Pvcglue.cloud.stage_env
         Pvcglue.cloud.stage_env = merged
@@ -75,35 +80,48 @@ module Pvcglue
     end
 
     def self.save_stage_env
-      Pvcglue::Packages.apply('env-set-stage'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
-      write_stage_env_cache
+      Pvcglue::Packages::Secrets.save_for_stage
+      # Pvcglue::Packages.apply('env-set-stage'.to_sym, :manager, Pvcglue::Manager.manager_node, 'pvcglue')
+      # write_stage_env_cache
     end
 
     def self.stage_env_defaults
-      {
-          'RAILS_SECRET_TOKEN' => SecureRandom.hex(64),
-          'DB_USER_POSTGRES_HOST' => db_host,
-          'DB_USER_POSTGRES_PORT' => "5432",
-          'DB_USER_POSTGRES_USERNAME' => "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}",
-          'DB_USER_POSTGRES_DATABASE' => "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}",
-          'DB_USER_POSTGRES_PASSWORD' => new_password,
-          'MEMCACHE_SERVERS' => memcached_host,
-          'REDIS_SERVER' => redis_host
-      }
+      defaults = {}
+
+      defaults['RAILS_SECRET_TOKEN'] = SecureRandom.hex(64) # From rails/railties/lib/rails/tasks/misc.rake
+      defaults['SECRET_KEY_BASE'] = SecureRandom.hex(64) # From rails/railties/lib/rails/tasks/misc.rake
+
+      if Pvcglue.cloud.minions_filtered('pg').any?
+        defaults['DB_USER_POSTGRES_HOST'] = db_host
+        defaults['DB_USER_POSTGRES_PORT'] = "5432"
+        defaults['DB_USER_POSTGRES_USERNAME'] = "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}"
+        defaults['DB_USER_POSTGRES_DATABASE'] = "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}"
+        defaults['DB_USER_POSTGRES_PASSWORD'] = new_password
+      end
+
+      if Pvcglue.cloud.minions_filtered('memcached').any?
+        defaults['MEMCACHE_SERVERS'] = memcached_host
+      end
+
+      if Pvcglue.cloud.minions_filtered('redis').any?
+        defaults['REDIS_SERVER'] = redis_host
+      end
+
+      defaults
     end
 
     def self.db_host
-      node = Pvcglue.cloud.find_node('db')
-      node['db']['private_ip']
+      # Assume 1 pg server
+      Pvcglue.cloud.minions_filtered('pg').values.first.private_ip
     end
 
     def self.memcached_host
-      node = Pvcglue.cloud.find_node('memcached', false)
+      node = Pvcglue.cloud.find_minion_by_name('memcached', false)
       node ? "#{node['memcached']['private_ip']}:11211" : ""
     end
 
     def self.redis_host
-      node = Pvcglue.cloud.find_node('redis', false)
+      node = Pvcglue.cloud.find_minion_by_name('redis', false)
       node ? "#{node['redis']['private_ip']}:6379" : ""
     end
 
@@ -143,7 +161,13 @@ module Pvcglue
       File.delete(stage_env_cache_file_name) if File.exists?(stage_env_cache_file_name)
     end
 
-
+    def self.apply_changes
+      Pvcglue.cloud.minions.each do |minion_name, minion|
+        if minion.has_roles? %w(web worker)
+          Pvcglue::Packages::Secrets.apply(minion)
+        end
+      end
+    end
   end
 
 end