pvcglue 0.1.39 → 0.9.0

data/lib/pvcglue/packages/postgresql.rb

@@ -1,10 +1,38 @@
-apt_package 'postgresql' do
-  action('start') { sudo 'service postgresql start' }
-  action('stop') { sudo 'service postgresql stop' }
-  action('restart') { sudo('service postgresql restart') }
-end
+module Pvcglue
+  class Packages
+    class Postgresql < Pvcglue::Packages
 
-package 'postgres' do
-  depends_on 'postgresql', 'libpq-dev'
-end
+      def installed?
+        get_minion_state(:postgresql_updated_at)
+      end
+
+      def install!
+        Pvcglue::Env.initialize_stage_env
+        connection.write_to_file_from_template(:root, 'postgresql.conf.erb', '/etc/postgresql/9.6/main/postgresql.conf', 'postgres', 'postgres', '0644')
+        connection.write_to_file_from_template(:root, 'pg_hba.conf.erb', '/etc/postgresql/9.6/main/pg_hba.conf', 'postgres', 'postgres', '0644')
+
+        # connection.run_get_stdout(:root, '', 'service postgresql restart')
+        # unless $?.exitstatus == 0
+        #   Pvcglue.logger.error { 'Unable to (re)start postgresql.  Getting status...' }
+        #   result = connection.run_get_stdout(:root, '', 'systemctl status postgresql.service')
+        #   puts result
+        #   raise('There was a problem restarting PostgreSQL.')
+        # end
+
+        username = Pvcglue.cloud.stage_env['DB_USER_POSTGRES_USERNAME']
+        password = Pvcglue.cloud.stage_env['DB_USER_POSTGRES_PASSWORD']
+        db_name = username # just for clarity in later statements.  This also must match database.yml.
+        # TODO:  Instead of ssh? below, use ssh!, but allow for the user having been already created.  It's only an issue for when the minion state has been reset
+        connection.ssh?(:root, '', %Q[cd /var/lib/postgresql && sudo -u postgres psql -c "CREATE ROLE #{username} LOGIN CREATEDB PASSWORD '#{password}';"])
+        connection.ssh!(:root, '', %Q[cd /var/lib/postgresql && sudo -u postgres psql -c "ALTER ROLE #{username} PASSWORD '#{password}' CREATEDB LOGIN"])
+        connection.ssh?(:root, '', %Q[cd /var/lib/postgresql && sudo -u postgres psql -c "CREATE DATABASE #{db_name} WITH OWNER #{username}"])
+        connection.ssh!(:root, '', %Q[cd /var/lib/postgresql && sudo -u postgres psql #{db_name} -c "ALTER SCHEMA public OWNER TO #{username}"])
 
+
+        set_minion_state(:postgresql_updated_at, Time.now.utc)
+      end
+
+    end
+
+  end
+end

data/lib/pvcglue/packages/roles.rb

@@ -0,0 +1,23 @@
+module Pvcglue
+  class Packages
+    class Roles < Pvcglue::Packages
+      def installed?
+        false
+      end
+
+      def post_install_check?
+        true
+      end
+
+      def install!
+        Pvcglue::Packages::Manager.apply(minion) if has_role?(:manager)
+        Pvcglue::Packages::LoadBalancer.apply(minion) if has_role?(:lb)
+        Pvcglue::Packages::Web.apply(minion) if has_role?(:web)
+        Pvcglue::Packages::Worker.apply(minion) if has_role?(:worker)
+        Pvcglue::Packages::Postgresql.apply(minion) if has_role?(:pg)
+        Pvcglue::Packages::Memcache.apply(minion) if has_role?(:mc)
+        Pvcglue::Packages::Redis.apply(minion) if has_role?(:redis)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/ruby.rb

@@ -0,0 +1,13 @@
+module Pvcglue
+  class Packages
+    class Ruby < Pvcglue::Packages
+      def installed?
+        connection.run_get_stdout!(user_name, '', 'rvm list strings') =~ /#{Pvcglue.configuration.ruby_version.gsub('.', '\.')}/
+      end
+
+      def install!
+        connection.run!(user_name, '', "rvm install #{Pvcglue.configuration.ruby_version}")
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/rvm.rb

@@ -1,78 +1,25 @@
-package 'rvm' do
-  depends_on 'curl'
-  depends_on 'rvm-bashrc'
+module Pvcglue
+  class Packages
+    class Rvm < Pvcglue::Packages
+      def installed?
+        connection.run_get_stdout!(user_name, '', 'type rvm | head -n 1') =~ /rvm is a function/
+      end
 
-  validate do
-    run('type rvm | head -n 1') =~ /rvm is a function/
-  end
+      def install!
+        connection.write_to_file_from_template(user_name, 'gemrc.erb', "/home/#{user_name}/.gemrc") # sets:  gem: --no-ri --no-rdoc
+        connection.write_to_file_from_template(user_name, 'web.bashrc.erb', "/home/#{user_name}/.bashrc")
 
-  apply do
-    run 'gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3'
-    run 'gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3' # Do it again, the first time only sets things up, and does not import the keys
-    # run 'curl -sSL https://rvm.io/mpapis.asc | gpg --import -'
-    run '\curl -sSL https://get.rvm.io | bash -s stable --with-default-gems="bundler"'
-    run "rvm requirements"
-  end
+        connection.run_get_stdout!(user_name, '', 'gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3')
+        # Do it again, the first time only sets things up, and does not import the keys
+        connection.run!(user_name, '', '\curl -sSL https://get.rvm.io | bash -s stable --with-default-gems=bundler')
 
-  remove do
-    run 'yes "yes" | rvm implode'
-  end
-end
+        # TODO: set autolibs mode so there are not installed automatically, as the user won't have sudo permissions later.
+        # OR create a 'install' user that can sudo...might be easier...
+        # Installing required packages: libreadline6-dev, libyaml-dev, libsqlite3-dev, sqlite3, autoconf, libgmp-dev, libgdbm-dev, libncurses5-dev, automake, libtool, bison, pkg-config, libffi-dev...............
 
-package 'rvm-bashrc' do
-  file({
-           :template => Pvcglue.template_file_name('web.bashrc.erb'),
-           :destination => '/home/deploy/.bashrc',
-           :create_dirs => false,
-           :permissions => 0644,
-           :user => 'deploy',
-           :group => 'deploy'
-       })
-end
+        connection.run_get_stdout!(user_name, '', 'rvm requirements')
 
-package 'gem' do
-  depends_on 'rvm-ruby'
-  action 'exists' do |gem_name|
-    run("gem list -i #{gem_name}") =~ /true/
-  end
-  action 'install' do |gem_name|
-    sudo "gem install #{gem_name} --no-ri --no-rdoc"
+      end
+    end
   end
-  action 'uninstall' do |gem_name|
-    sudo "gem uninstall #{gem_name} -x -a"
-  end
-end
-
-package 'bundler' do
-  depends_on 'gem'
-  apply { trigger 'gem:install', 'bundler' }
-  remove { trigger 'gem:remove', 'bundler' }
-  validate { trigger 'gem:exists', 'bundler' }
 end
-
-package 'rvm-ruby' do
-  depends_on 'rvm'
-
-  validate do
-    run('rvm list strings') =~ /#{Pvcglue.configuration.ruby_version.gsub('.', '\.')}/
-  end
-
-  apply do
-    run "rvm install #{Pvcglue.configuration.ruby_version}"
-    # run "rvm --default use 2.0.0"
-  end
-
-  remove do
-    run "rvm remove --archive --gems #{Pvcglue.configuration.ruby_version}"
-  end
-
-end
-
-package 'no-rdoc' do
-  file({
-           :template => Pvcglue.template_file_name('gemrc.erb'),
-           :destination => '/home/deploy/.gemrc',
-           :create_dirs => false
-       })
-end
-

data/lib/pvcglue/packages/secrets.rb

@@ -0,0 +1,36 @@
+module Pvcglue
+  class Packages
+    class Secrets < Pvcglue::Packages
+      def installed?
+        false
+      end
+
+      def install!
+        Pvcglue::Env.initialize_stage_env
+        connection.write_to_file_from_template(user_name, 'web.env.erb', Pvcglue.cloud.env_file_name, nil, nil, '0640') # TODO:  Double check permissions
+        restart_web_app!
+      end
+
+      def post_install_check?
+        true
+      end
+
+      def restart_web_app!
+        if connection.file_exists?(user_name, Pvcglue.cloud.deploy_to_app_current_temp_dir)
+          connection.ssh!(user_name, '', "touch #{Pvcglue.cloud.restart_txt_file_name}")
+        end
+      end
+
+      def self.load_for_stage
+        data = Pvcglue::Packages::Manager.new.load_secrets
+        data = '' if data.nil?
+        Pvcglue.cloud.stage_env = TOML.parse(data)
+      end
+
+      def self.save_for_stage
+        data = TOML.dump(Pvcglue.cloud.stage_env)
+        Pvcglue::Packages::Manager.new.save_secrets(data)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/ssh_key_check.rb

@@ -0,0 +1,11 @@
+module Pvcglue
+  class Packages
+    class SshKeyCheck < Pvcglue::Packages
+      def installed?
+        # This has the side effect of adding the server to known_hosts file, to prevent needing interactive prompt
+        connection.ssh_retry_wait(:root, '-o strictHostKeyChecking=no', 'echo', 30, 1)
+        true
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/ssl.rb

@@ -0,0 +1,45 @@
+module Pvcglue
+  class Packages
+    class Ssl < Pvcglue::Packages
+      def installed?
+        return true if Pvcglue.cloud.ssl_mode == :none
+        false
+      end
+
+      def install!
+        # TODO:  Support using already created certs
+        return true unless Pvcglue.cloud.ssl_mode == :acme
+
+        Pvcglue::Packages::SslAcme.apply(minion)
+
+        # Issue Certificate
+        first_domain = Pvcglue.cloud.domains.first
+        domains = Pvcglue.cloud.domains.map { |domain| "-d #{domain}" }
+        first_domain_option = domains.first
+        domain_options = domains.join(' ')
+        staging_option = Pvcglue.command_line_options[:create_test_cert] ? '--staging ' : ''
+        force_option = Pvcglue.command_line_options[:force_cert] ? '--force ' : ''
+        debug_option = Pvcglue.logger.level == 0 ? '--debug ' : ''
+
+        unless Net::HTTP.get(first_domain, '/.well-known/acme-challenge/test.html') =~ /shiny/
+          Pvcglue.logger.error("Unable to connect to #{first_domain} at #{minion.public_ip}")
+          raise(Thor::Error, 'Please fix and then restart.')
+        end
+
+        result = connection.ssh?(:root, '', "/root/.acme.sh/acme.sh #{debug_option}#{staging_option}#{force_option}--issue #{domain_options} -w #{Pvcglue.cloud.letsencrypt_root}")
+        raise result.inspect unless result.exitstatus == 0 || result.exitstatus == 2
+
+        # Install Certificate
+        connection.mkdir_p(:root, Pvcglue.cloud.nginx_config_ssl_path)
+        # acme.sh --installcert -d theos.in --keypath /etc/nginx/ssl/theos.in/theos.in.key --fullchainpath /etc/nginx/ssl/theos.in/theos.in.cer --reloadcmd 'systemctl reload nginx'
+
+        connection.ssh!(:root, '', "/root/.acme.sh/acme.sh --installcert #{first_domain_option} --keypath #{Pvcglue.cloud.nginx_ssl_key_file_name} --fullchainpath #{Pvcglue.cloud.nginx_ssl_crt_file_name}  --reloadcmd 'systemctl reload nginx'")
+      end
+
+      def post_install_check?
+        true
+      end
+
+    end
+  end
+end

data/lib/pvcglue/packages/ssl_acme.rb

@@ -0,0 +1,29 @@
+module Pvcglue
+  class Packages
+    class SslAcme < Pvcglue::Packages
+      def installed?
+        return true if Pvcglue.cloud.ssl_mode == :none
+        get_minion_state(:acme_sh_updated_at)
+      end
+
+      def install!
+        # Thanks to https://www.rmedgar.com/blog/using-acme-sh-with-nginx
+
+        connection.ssh!(:root, '', 'curl https://get.acme.sh | sh')
+
+        # mkdir -p /var/www/le_root/.well-known/acme-challenge
+        connection.mkdir_p(:root, Pvcglue.cloud.letsencrypt_full)
+        #chown -R root:www-data /var/www/le_root
+        connection.chown(:root, Pvcglue.cloud.letsencrypt_root, 'root', 'www-data', '-R')
+
+        # Test with http://www.example.com/.well-known/acme-challenge/test.html
+        connection.write_to_file(:root, "Everything's shiny, Cap'n. Not to fret.", File.join(Pvcglue.cloud.letsencrypt_full, 'test.html'), 'root', 'www-data', '660')
+
+        connection.ssh!(:root, '', 'systemctl reload nginx.service')
+
+        set_minion_state(:acme_sh_updated_at, Time.now.utc)
+      end
+
+    end
+  end
+end

data/lib/pvcglue/packages/swap.rb

@@ -0,0 +1,14 @@
+module Pvcglue
+  class Packages
+    class Swap < Pvcglue::Packages
+      def installed?
+
+        # sudo("cat /etc/fstab") =~ /\/swapfile/
+        true
+      end
+
+      def install!
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/unattended_upgrades.rb

@@ -0,0 +1,20 @@
+module Pvcglue
+  class Packages
+    class UnattendedUpgrades < Pvcglue::Packages
+      def installed?
+        get_minion_state(:installed_unattended_upgrades_at)
+      end
+
+      def install!
+        connection.write_to_file_from_template(:root, '20auto-upgrades.erb', '/etc/apt/apt.conf.d/20auto-upgrades')
+        connection.write_to_file_from_template(:root, '50unattended-upgrades.erb', '/etc/apt/apt.conf.d/50unattended-upgrades')
+      end
+
+      def post_install_check?
+        connection.run!(:root, '', 'service unattended-upgrades restart')
+        set_minion_state(:installed_unattended_upgrades_at, Time.now.utc)
+        true
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/users.rb

@@ -0,0 +1,20 @@
+module Pvcglue
+  class Packages
+    class Users < Pvcglue::Packages
+      def installed?
+        result = connection.run_get_stdout(:root, '', "getent passwd #{user_name} && groups #{user_name}")
+        result =~ /^#{user_name}:/ && result =~ /#{user_name} sudo/
+      end
+
+      def install!
+        # connection.run!(:root, '', 'mkdir -p ~/.pvc && chmod 600 ~/.pvc')  # TODO:  Still needed?
+        # connection.run!(:root, '', "useradd -d /home/#{user_name} -G sudo -m -U #{user_name} && usermod -s /bin/bash #{user_name}")
+
+        connection.run!(:root, '', "useradd -d /home/#{user_name} -G sudo -m -U #{user_name}")
+        connection.run!(:root, '', "usermod -s /bin/bash #{user_name}")
+        # TODO:  Lock down the sudo permissions to just let the user deploy
+        connection.run!(:root, '', "echo '#{user_name} ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers")
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/web.rb

@@ -0,0 +1,50 @@
+module Pvcglue
+  class Packages
+    class Web < Pvcglue::Packages
+
+      def installed?
+        false
+      end
+
+      def install!
+        Pvcglue::Packages::DirBase.apply(minion)
+        Pvcglue::Packages::DirShared.apply(minion)
+        Pvcglue::Packages::Rvm.apply(minion)
+        Pvcglue::Packages::Ruby.apply(minion)
+        Pvcglue::Packages::Bundler.apply(minion)
+        Pvcglue::Packages::Secrets.apply(minion)
+        connection.write_to_file_from_template(:root, 'web.nginx.conf.erb', '/etc/nginx/nginx.conf')
+
+        set_passenger_ruby # needs to be set before rendering 'web.sites-enabled.erb'
+        connection.write_to_file_from_template(:root, 'web.sites-enabled.erb', "/etc/nginx/sites-enabled/#{Pvcglue.cloud.app_and_stage_name}")
+
+
+      end
+
+      def post_install_check?
+        result = connection.run_get_stdout(:root, '', 'service nginx restart')
+        if $?.exitstatus == 0
+          # Pvcglue.logger.debug { result }
+          true
+        else
+          Pvcglue.logger.error { 'Unable to (re)start nginx.  Getting status...' }
+          result = connection.run_get_stdout(:root, '', 'systemctl status nginx.service')
+          puts result
+          false
+        end
+
+        # TODO:  Ping the server as a double check.
+      end
+
+      def set_passenger_ruby
+        info = connection.run_get_stdout!(user_name, '', "rvm use #{Pvcglue.configuration.ruby_version} && $(which passenger-config) --ruby-command")
+        if info =~ /passenger_ruby (.*)/
+          Pvcglue.cloud.passenger_ruby = $1
+        else
+          raise "'passenger_ruby' not found." unless Pvcglue.cloud.passenger_ruby
+        end
+
+      end
+    end
+  end
+end

data/lib/pvcglue/stack.rb

@@ -0,0 +1,166 @@
+module Pvcglue
+  class Stack
+    def self.build(minions, roles_filter)
+      Pvcglue::Stack.new(roles_filter).run(minions)
+    end
+
+    def initialize(roles_filter)
+      @roles_filter = roles_filter
+    end
+
+    def apply_role?(role)
+      @roles_filter == 'all' || role == @roles_filter
+    end
+
+    def run(minions)
+      # puts "Configuring nodes for #{@roles_filter}."
+
+      # ap Pvcglue.configuration
+      # ap Pvcglue.cloud.data
+      # ap Pvcglue.cloud.app_name
+      # ap Pvcglue.cloud.project
+      # ap Pvcglue.cloud.project.name
+      # ap Pvcglue.cloud.project.stages.map { |stage| stage.to_dot.stage }
+      # ap Pvcglue.cloud.project.stages.map { |stage| stage.name }
+      # ap Pvcglue.cloud.stage
+      # ap Pvcglue.cloud.minions
+      # ap Pvcglue.cloud.minions.map { |key, value| value.roles }
+      # minions.each do |minion_name, minion|
+      #   ap minion_name
+      #   ap minion.first
+      # end
+      # ap minions['staging-pg'].private_ip
+      # minions['staging-pg'].private_ip = '127.0.0.1'
+      # ap minions['staging-pg'].private_ip
+      new_minions = []
+      minions.each do |minion_name, minion|
+        Pvcglue.logger_current_minion = minion
+        # droplet = Pvcglue::DigitalOcean.client.droplets.find(id: 38371925)
+        # minion.droplet = droplet
+        # new_minions << minion if true || minion.provision!
+        unless minion.provisioned?
+          if droplets.detect { |droplet| droplet.name == minion_name }
+            raise(Thor::Error, "Machine with the name of '#{minion_name}' already exists!")
+          end
+          minion.provision!
+          new_minions << minion
+        end
+      end
+      Pvcglue.logger_current_minion = nil
+
+      if new_minions.size > 0
+        # ap new_minions
+        Pvcglue.logger.info("Checking status of new minions (#{new_minions.size})...")
+        time = Benchmark.realtime do
+          begin
+            lazy_minions = get_lazy_minions(minions)
+            unless lazy_minions.size == 0
+              # puts '*'*175
+              # ap waiting_for
+              Pvcglue.logger.info("Waiting for #{lazy_minions.map { |key, value| key }.join(', ')}...")
+              sleep(1.5)
+            end
+          end until lazy_minions.size == 0
+          updated_minions = write_config(new_minions)
+          updated_minions.each do |updated_minion|
+            minions[updated_minion.machine_name].private_ip = updated_minion.private_ip
+            minions[updated_minion.machine_name].public_ip = updated_minion.public_ip
+            minions[updated_minion.machine_name].cloud_id = updated_minion.cloud_id
+            minions[updated_minion.machine_name].connection = Pvcglue::Connection.new(minions[updated_minion.machine_name])
+          end
+        end
+        Pvcglue.logger.info("Minions (finally) ready after #{time.round(2)} seconds!")
+        # Pvcglue.cloud.reload_minions!
+      end
+
+      minions.each do |minion_name, minion|
+        next unless minion.has_role?(@roles_filter)
+        Pvcglue.logger_current_minion = minion
+        minion.build!
+      end
+      Pvcglue.logger_current_minion = nil
+
+      # raise(Thor::Error, 'STOP!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+      #
+      # %w(lb db web caching redis).each do |role|
+      #   if apply_role?(role)
+      #     Pvcglue::Packages.apply(role.to_sym, :build, Pvcglue.cloud.minions_filtered(role))
+      #   end
+      # end
+
+      Pvcglue::Pvcify.run unless minions.values.first.minion_manager?
+
+    end
+
+    def write_config(minions)
+      # TODO:  Just read and write the configuration from manager without writing to `minion.cloud.local_file_name`
+      Pvcglue::Manager.pull_configuration
+      data = File.read(Pvcglue.cloud.local_file_name)
+      updated_minions = minions.map do |minion|
+        # puts '-'*175
+        # ap minion.droplet
+        # Refresh droplet (didn't find a reload method)
+        droplet_id = minion.droplet.id.to_s
+        # ap droplet_id
+        minion.droplet = Pvcglue::DigitalOcean.client.droplets.find(id: droplet_id)
+        # puts '='*175
+        # ap minion.droplet
+        ip_addresses = Pvcglue::DigitalOcean.get_ip_addresses(minion.droplet)
+        data = update_minion_data(minion, ip_addresses, droplet_id, data)
+        minion.public_ip = ip_addresses.public
+        minion.private_ip = ip_addresses.private
+        minion.cloud_id = droplet_id
+        minion
+      end
+      File.write(Pvcglue.cloud.local_file_name, data)
+      Pvcglue::Manager.push_configuration
+      updated_minions
+    end
+
+    def update_minion_data(minion, ip_addresses, cloud_id, data)
+      unless minion.public_ip_address.nil? && minion.private_id_address.nil? && minion.cloud_id.nil?
+        raise(Thor::Error, "#{minion.machine_name} has previously defined ip address(es) or cloud_id, can not change.")
+      end
+      if ip_addresses.public.nil? || ip_addresses.private.nil? || cloud_id.nil?
+        raise(Thor::Error, "New IP addresses (#{ip_addresses}) or cloud_id (#{cloud_id}) are not valid.")
+      end
+
+
+      replacement = "\\1\\2\n\\1public_ip = '#{ip_addresses.public}'\n\\1private_ip = '#{ip_addresses.private}'\n\\1cloud_id = '#{cloud_id}'"
+      new_data = data.sub(/( *)(name\s*=\s*['"]#{Regexp.quote(minion.machine_name)}['"])/, replacement)
+      raise "Unable to update minion data for #{minion.machine_name}." if data == new_data
+
+      # replacement = "$1$2\n$1public_ip = '#{ip_addresses.public}'\n$1private_ip = '#{ip_addresses.private}'"
+      # new_data = data.sub(/( *)(name\s*=\s*['"]staging-lb['"])/) do |match|
+      #   "#{$1}#{$2}\n#{$1}public_ip = '#{ip_addresses.public}'\n#{$1}private_ip = '#{ip_addresses.private}'"
+      # end
+      # puts new_data
+
+
+      Pvcglue.logger.debug("Updated configuration for machine named #{minion.machine_name}.")
+      new_data
+    end
+
+    def droplets
+      @droplets ||= Pvcglue::DigitalOcean.client.droplets.all
+    end
+
+    def get_lazy_minions(minions)
+      droplets = Pvcglue::DigitalOcean.client.droplets.all
+
+      minions.select do |minion_name, minion|
+        Pvcglue.logger_current_minion = minion
+        # next unless minion.machine_name == 'staging-lb'
+        # ap minion_name
+        found = droplets.detect do |droplet|
+          droplet[:name] == minion.machine_name
+        end
+        # ap found
+        # ap found[:status]
+        !found || found[:status] != 'active'
+      end
+      # ap minions
+    end
+
+  end
+end