pvcglue 0.1.39 → 0.9.0

data/lib/pvcglue/packages/apt.rb

@@ -0,0 +1,74 @@
+module Pvcglue
+  class Packages
+    class Apt < Pvcglue::Packages
+      WEB_WORKER_PACKAGES = %w[
+        build-essential
+        git
+        git-core
+        libpq-dev
+        libxml2
+        libxml2-dev
+        imagemagick
+        passenger
+        nginx
+        nginx-extras
+        nodejs
+      ]
+      PACKAGES = {
+          common: %w[
+        htop
+        ufw
+        unattended-upgrades
+        curl
+        ncdu
+          ],
+          manager: %w[
+            git
+            git-core
+          ],
+          lb: %w[
+            nginx
+            nginx-extras
+          ],
+          web: WEB_WORKER_PACKAGES,
+          worker: WEB_WORKER_PACKAGES,
+          pg: %w[
+            postgresql-9.6
+            postgresql-contrib-9.6
+            libpq-dev
+          ],
+          mc: %w[
+          ],
+      }
+
+      def installed?
+        false # just let apt take care of this for now
+      end
+
+      def install!
+        connection.run!(:root, '', "DEBIAN_FRONTEND=noninteractive apt install -y #{get_package_list}")
+      end
+
+      def post_install_check?
+        true
+      end
+
+      def get_package_list
+        get_packages.join(' ')
+      end
+
+      def all_packages
+        @all_packages ||= PACKAGES.with_indifferent_access
+      end
+
+      def get_packages
+        packages = all_packages[:common]
+        minion.roles.each do |role|
+          packages += all_packages[role] if all_packages[role]
+        end
+        packages
+      end
+
+    end
+  end
+end

data/lib/pvcglue/packages/apt_repos.rb

@@ -0,0 +1,48 @@
+module Pvcglue
+  class Packages
+    class AptRepos < Pvcglue::Packages
+      PASSENGER_SOURCES_LIST_FILENAME = '/etc/apt/sources.list.d/passenger.list'
+      PASSENGER_SOURCES_LIST_DATA = 'deb https://oss-binaries.phusionpassenger.com/apt/passenger xenial main'
+
+      def nginx_needed?
+        has_roles? %w(lb web)
+      end
+
+      def node_js_needed?
+        has_roles? %w(web worker)
+      end
+
+      def postgresql_needed?
+        has_roles? %w(pg)
+      end
+
+      def installed?
+        get_minion_state(:apt_repos_updated_at)
+      end
+
+      def install!
+        # These could be refactored into packages.  :)
+
+        if nginx_needed?
+          # Reference:  https://www.phusionpassenger.com/library/install/nginx/install/oss/xenial/
+          connection.run!(:root, '', 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7')
+          connection.run!(:root, '', 'apt-get install -y apt-transport-https ca-certificates')
+          connection.write_to_file(:root, PASSENGER_SOURCES_LIST_DATA, PASSENGER_SOURCES_LIST_FILENAME)
+        end
+
+        if node_js_needed?
+          # Reference:  http://tecadmin.net/install-latest-nodejs-npm-on-ubuntu/
+          connection.run!(:root, '', 'apt-get install -y apt-transport-https ca-certificates python-software-properties lsb-release')
+          connection.run!(:root, '', 'curl -sL https://deb.nodesource.com/setup_7.x | bash -')
+        end
+
+        if postgresql_needed?
+          connection.run!(:root, '', 'add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main"')
+          connection.run!(:root, '', 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -')
+        end
+
+        set_minion_state(:apt_repos_updated_at, Time.now.utc)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/apt_update.rb

@@ -0,0 +1,18 @@
+module Pvcglue
+  class Packages
+    class AptUpdate < Pvcglue::Packages
+      def installed?
+        # TODO:  Add a "force" option
+        updated_at = get_minion_state(:last_apt_updated_at)
+        return false unless updated_at
+        true
+        # updated_at > Time.now.utc - 8.hours # Unattended upgrades should take care of refreshing this automatically
+      end
+
+      def install!
+        connection.run!(:root, '', 'DEBIAN_FRONTEND=noninteractive apt-get update -y -qq')
+        set_minion_state(:last_apt_updated_at, Time.now.utc)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/apt_upgrade.rb

@@ -0,0 +1,20 @@
+module Pvcglue
+  class Packages
+    class AptUpgrade < Pvcglue::Packages
+      def installed?
+        # TODO:  Add a "force" option
+        updated_at = get_minion_state(:apt_upgraded_at)
+        return false unless updated_at
+
+        # updated_at > Time.now.utc - 8.hours
+        # TODO:  Give the user a warning after a period of time
+        true
+      end
+
+      def install!
+        minion.connection.run!(:root, '', 'DEBIAN_FRONTEND=noninteractive apt-get upgrade -y')
+        set_minion_state(:apt_upgraded_at, Time.now.utc)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/authorized_keys.rb

@@ -0,0 +1,33 @@
+module Pvcglue
+  class Packages
+    class AuthorizedKeys < Pvcglue::Packages
+      def installed?
+        false
+      end
+
+      def post_install_check?
+        true
+      end
+
+      def install!
+        # TODO:  Safety check to see if user is locking himself out.  :)
+        data = minion.get_root_authorized_keys_data
+        if data.count == 0
+          raise('No authorized keys found for root users!')
+          # TODO:  work out system for pvc-manager access
+          data = [`cat ~/.ssh/id_rsa.pub`.strip]
+        end
+        connection.write_to_file(:root, data.join("\n"), '/root/.ssh/authorized_keys')
+
+        connection.mkdir_p(:root, "/home/#{user_name}/.ssh", user_name, user_name, '0700')
+        data = minion.get_users_authorized_keys_data
+        if data.count == 0
+          raise('No authorized keys found for users!')
+          # TODO:  work out system for pvc-manager access
+          data = [`cat ~/.ssh/id_rsa.pub`.strip]
+        end
+        connection.write_to_file(:root, data.join("\n"), "/home/#{user_name}/.ssh/authorized_keys", user_name, user_name, '0644')
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/bundler.rb

@@ -0,0 +1,14 @@
+module Pvcglue
+  class Packages
+    class Bundler < Pvcglue::Packages
+      def installed?
+        connection.run_get_stdout!(user_name, '', 'which bundler') =~ /bundler/
+      end
+
+      def install!
+        # TODO:  Figure out why this isn't automatic
+        connection.run!(user_name, '', 'gem install bundler')
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/dir_base.rb

@@ -0,0 +1,16 @@
+module Pvcglue
+  class Packages
+    class DirBase < Pvcglue::Packages
+      def installed?
+        result = connection.run_get_stdout(user_name, '', "stat --format=%U:%G:%a #{Pvcglue.cloud.web_app_base_dir}").strip
+        result == "#{user_name}:#{user_name}:2755"
+      end
+
+      def install!
+        dir = Pvcglue.cloud.web_app_base_dir
+        # used following as a guide for next line: http://capistranorb.com/documentation/getting-started/authentication-and-authorisation/
+        connection.run!(user_name, '', "mkdir -p #{dir} && chown #{user_name}:#{user_name} #{dir} && chmod 0755 #{dir} && umask 0002 && chmod g+s #{dir}")
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/dir_shared.rb

@@ -0,0 +1,16 @@
+module Pvcglue
+  class Packages
+    class DirShared < Pvcglue::Packages
+      def installed?
+        result = connection.run_get_stdout(user_name, '', "stat --format=%U:%G:%a #{Pvcglue.cloud.deploy_to_app_shared_dir}").strip
+        result == "#{user_name}:#{user_name}:2755"
+      end
+
+      def install!
+        dir = Pvcglue.cloud.deploy_to_app_shared_dir
+        # used following as a guide for next line: http://capistranorb.com/documentation/getting-started/authentication-and-authorisation/
+        connection.run!(user_name, '', "mkdir -p #{dir} && chown #{user_name}:#{user_name} #{dir} && chmod 0755 #{dir} && umask 0002 && chmod g+s #{dir}")
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/firewall.rb

@@ -1,48 +1,32 @@
-# Reference:  http://manpages.ubuntu.com/manpages/precise/en/man8/ufw-framework.8.html
-package 'firewall-config' do
-
-  file({
-           :template => Pvcglue.template_file_name('ufw.rules6.erb'),
-           :destination => '/lib/ufw/user6.rules',
-           :create_dirs => false,
-           :permissions => 0640,
-           :user => 'root',
-           :group => 'root'
-       }) {  }
-
-  file({
-           :template => Pvcglue.template_file_name('ufw.rules.erb'),
-           :destination => '/lib/ufw/user.rules',
-           :create_dirs => false,
-           :permissions => 0640,
-           :user => 'root',
-           :group => 'root'
-       }) { sudo('service ufw restart') }
-
-end
-
-package 'firewall-enabled' do
-  validate do
-    result = sudo('ufw status verbose')
-    result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
-  end
-
-  apply do
-    sudo('ufw --force enable')
-  end
-end
-
-# TODO:  add command line command for this
-package 'update-firewall' do
-  # quick update of firewall settings only.  Full bootstrap must be performed first.
-  depends_on 'firewall-config'
-  depends_on 'firewall-enabled'
-end
-
-
-# TODO:  add command line command for this
-package 'firewall-status' do
-  apply do
-    run "ufw status verbose"
+module Pvcglue
+  class Packages
+    class Firewall < Pvcglue::Packages
+      # Reference:  http://manpages.ubuntu.com/manpages/xenial/en/man8/ufw-framework.8.html
+      # Examples:  https://help.ubuntu.com/community/UFW
+      def installed?
+        result = connection.run_get_stdout!(:root, '', 'ufw status verbose')
+        result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
+      end
+
+      def install!
+        connection.run!(:root, '', 'ufw disable; ufw --force reset; ufw allow ssh; ufw --force enable')
+        # connection.run!(:root, '', 'ufw logging off')
+        connection.run!(:root, '', 'ufw logging low')
+
+        if has_role?(:lb)
+          connection.run!(:root, '', 'ufw allow http')
+          connection.run!(:root, '', 'ufw allow https')
+        end
+
+        unless has_role?(:manager)
+          minion.cloud.minions.each do |other_minion_name, other_minion|
+            next if other_minion_name == minion.machine_name
+            connection.run!(:root, '', "ufw allow from #{other_minion.private_ip}")
+          end
+        end
+
+      end
+
+    end
   end
 end

data/lib/pvcglue/packages/load_balancer.rb

@@ -0,0 +1,71 @@
+module Pvcglue
+  class Packages
+    class LoadBalancer < Pvcglue::Packages
+      def installed?
+        false
+      end
+
+      def install!
+        if Pvcglue.cloud.ssl_mode == :acme && !connection.file_exists?(:root, Pvcglue.cloud.nginx_ssl_crt_file_name)
+          # Don't include the SSL stuff in the Nginx config until we have a cert,
+          # but Nginx has to be configured to get the cert from Let's Encrypt
+          Pvcglue.cloud.set_ssl_mode_override(:none)
+        end
+
+        apply_nginx_includes
+        apply_nginx_config
+        nginx_restart!
+
+        Pvcglue::Packages::Ssl.apply(minion)
+
+        if Pvcglue.cloud.ssl_mode_override
+          Pvcglue.cloud.set_ssl_mode_override(nil)
+          apply_nginx_config
+          nginx_restart!
+        end
+
+        sync_maintenance_files
+      end
+
+      def nginx_restart!
+        result = connection.run_get_stdout(:root, '', 'service nginx restart')
+        if $?.exitstatus == 0
+          # Pvcglue.logger.debug { result }
+          # true
+        else
+          Pvcglue.logger.error { 'Unable to (re)start nginx.  Getting status...' }
+          result = connection.run_get_stdout(:root, '', 'systemctl status nginx.service')
+          puts result
+          raise
+        end
+      end
+
+      def apply_nginx_includes
+        connection.mkdir_p(:root, '/etc/nginx/includes')
+        connection.write_to_file_from_template(:root, 'letsencrypt-webroot.erb', '/etc/nginx/includes/letsencrypt-webroot')
+      end
+
+      def apply_nginx_config
+        connection.write_to_file_from_template(:root, 'lb.nginx.conf.erb', '/etc/nginx/nginx.conf')
+        connection.write_to_file_from_template(:root, 'lb.sites-enabled.erb', "/etc/nginx/sites-enabled/#{Pvcglue.cloud.app_and_stage_name}")
+      end
+
+      def post_install_check?
+        # TODO:  Ping the server as a double check.
+        true
+      end
+
+      def sync_maintenance_files
+        source_dir = Pvcglue.configuration.app_maintenance_files_dir
+        dest_dir = Pvcglue.cloud.maintenance_files_dir
+        maintenance_file_name = File.join(source_dir, 'maintenance.html')
+        unless File.exists?(maintenance_file_name)
+          # TODO:  Make this use a template
+          `mkdir -p #{source_dir}`
+          File.write(maintenance_file_name, '-Maintenance Mode-')
+        end
+        connection.rsync_up(user_name, '-rzv --exclude=maintenance.on --delete', source_dir, dest_dir, mkdir = true)
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/maintenance_mode.rb

@@ -0,0 +1,28 @@
+module Pvcglue
+  class Packages
+    class MaintenanceMode < Pvcglue::Packages
+      def installed?
+        false
+      end
+
+      def install!
+        if options[:maintenance_mode] == 'on'
+          connection.run!(user_name, '', "touch #{Pvcglue.cloud.maintenance_mode_file_name}")
+        elsif options[:maintenance_mode] == 'off'
+          result = connection.run?(user_name, '', "rm #{Pvcglue.cloud.maintenance_mode_file_name}")
+          if result.exitstatus == 1
+            Pvcglue.logger.warn('Maintenance mode was already off.')
+          elsif result.exitstatus != 0
+            raise result.inspect
+          end
+        else
+          raise("Invalid maintenance_mode option:  #{options[:maintenance_mode]}")
+        end
+      end
+
+      def post_install_check?
+        true
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/manager.rb

@@ -1,116 +1,118 @@
-require 'toml'
-apt_package 'htop'
-apt_package 'ufw'
-
-package 'bootstrap-manager' do
-  # TODO: firewall and ssh port config
-  depends_on 'authenticate-host'
-  depends_on 'htop'
-  # depends_on 'ufw'
-  #depends_on 'deploy-user'
-  #depends_on 'sshd-config'
-  #depends_on 'firewall-config'
-  depends_on 'pvcglue-user'
-  depends_on 'manager-copy-id'
-end
+module Pvcglue
+  class Packages
+    class Manager < Pvcglue::Packages
 
-package 'authenticate-host' do
-  apply do
-    sudo "ls" # side-effect will add host to known_hosts
-  end
-end
+      def initialize(minion = nil, options = {})
+        minion = Pvcglue.cloud.manager_minion if minion.nil?
+        super
+      end
 
-package 'pvcglue-user' do
-  apply do
-    # Local variables used to improve readability of bash commands :)
-    user_name = Pvcglue::Manager.user_name
-    home_dir = Pvcglue::Manager.home_dir
-    manager_dir = Pvcglue::Manager.manager_dir
-    ssh_dir = Pvcglue::Manager.ssh_dir
-
-    sudo "useradd -d #{home_dir} -m -U #{user_name}"
-    sudo "usermod -s /bin/bash #{user_name}"
-    sudo "mkdir -p #{manager_dir} && chown #{user_name}:#{user_name} #{manager_dir} && chmod 700 #{manager_dir}"
-    sudo "mkdir -p #{ssh_dir} && chown #{user_name}:#{user_name} #{ssh_dir} && chmod 700 #{ssh_dir}"
-  end
+      def installed?
+        get_minion_state(:manager_installed_at)
+      end
 
-  remove do
-    raise "removing user not supported, yet.  It needs some 'Are you *really* sure?' stuff."
-    # user_name = Pvcglue::Manager.user_name
-    # home_dir = Pvcglue::Manager.home_dir
-    #sudo "userdel -f #{user_name}"
-    #sudo "rm -rf #{home_dir}"
-  end
+      def install!
+        connection.mkdir_p(user_name, manager_dir, nil, nil, 700)
 
-  validate do
-    user_name = Pvcglue::Manager.user_name
-    # home_dir = Pvcglue::Manager.home_dir
-    #sudo "userdel -f #{user_name}"; sudo "rm -rf #{home_dir}"; raise "User has been deleted"
-    sudo("getent passwd #{user_name}") =~ /^#{user_name}:/
-  end
-end
+        connection.ssh!(user_name, '', 'git config --global user.name pvc_manager')
+        connection.ssh!(user_name, '', 'git config --global --global user.email pvc_manager@pvc.local')
+        connection.ssh!(user_name, '', "git init #{Pvcglue::Manager.manager_dir}")
 
-package 'manager-copy-id' do
-  validate do
-    authorized_keys_file_name = Pvcglue::Manager.authorized_keys_file_name
-    user_key = `cat ~/.ssh/id_rsa.pub`.strip
-    auth = run("cat #{authorized_keys_file_name}")
-    auth.include?(user_key)
-  end
+        # TODO:  Create example configuration file, if none exists
+        connection.write_to_file(user_name, "#{Time.now.utc.to_s}\n", manager_test_filename)
+        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git status))
+        git_commit!
+        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git status))
 
-  apply do
-    authorized_keys_file_name = Pvcglue::Manager.authorized_keys_file_name
-    user_name = Pvcglue::Manager.user_name
-    copy_id = %Q[cat ~/.ssh/id_rsa.pub | ssh -o BatchMode=yes -o StrictHostKeyChecking=no #{node.get(:user)}@#{node.host} "cat >> #{authorized_keys_file_name}"]
-    system "#{copy_id}"
-    run(%Q[cat "" >> #{authorized_keys_file_name}])
-    sudo "chown #{user_name}:#{user_name} #{authorized_keys_file_name} && chmod 600 #{authorized_keys_file_name}"
-  end
-end
+        # https://git-scm.com/book/en/v2/Git-Basics-Viewing-the-Commit-History
+        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git log --pretty=format:"%h - %an, %ar : %s" && echo))
 
-package 'manager-push' do
-  apply do
-    if File.exists?(::Pvcglue.cloud.local_file_name)
-      # scp foobar.txt your_username@remotehost.edu:/some/remote/directory
-      cmd = %{scp #{::Pvcglue.cloud.local_file_name} #{node.get(:user)}@#{node.host}:#{::Pvcglue::Manager.manager_file_name}}
-      puts "Running `#{cmd}`"
+        set_minion_state(:manager_installed_at, Time.now.utc)
+      end
 
-      unless system cmd
-        raise(Thor::Error, "Error:  #{$?}")
+      def post_install_check?
+        return false unless connection.file_exists?(user_name, manager_dir)
+        return false unless working_directory_clean?
+        connection.file_exists?(user_name, manager_test_filename)
       end
 
-      run(%Q[chmod 600 #{::Pvcglue::Manager.manager_file_name}])
-    else
-      puts "Local file not found:  #{::Pvcglue.cloud.local_file_name}"
-    end
-  end
-end
+      def manager_dir
+        Pvcglue::Manager.manager_dir
+      end
 
-package 'manager-pull' do
-  apply do
-    # scp your_username@remotehost.edu:foobar.txt /some/local/directory
-    cmd = %{scp #{node.get(:user)}@#{node.host}:#{::Pvcglue::Manager.manager_file_name} #{::Pvcglue.cloud.local_file_name}}
-    puts "Running `#{cmd}`"
+      def manager_test_filename
+        File.join(manager_dir, 'install.log')
+      end
 
-    unless system cmd
-      raise(Thor::Error, "Error:  #{$?}")
-    end
+      def self.get_configuration
+        new.get_configuration
+      end
 
-    puts "Saved as:  #{::Pvcglue.cloud.local_file_name}"
-  end
-end
+      def get_configuration
+        # if connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
+        #   data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
+        # else
+        #   # raise(Thor::Error, "Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}")
+        #   raise("Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}")
+        # end
+        data = '' # to use `data` in block
+        Pvcglue.filter_verbose do
+          data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
+        end
+        ::Pvcglue.cloud.data = TOML.parse(data)
+      end
+
+      def self.push_configuration
+        new.push_configuration
+      end
+
+      def push_configuration
+        connection.upload_file(user_name, ::Pvcglue.cloud.local_file_name, ::Pvcglue::Manager.manager_file_name, nil, nil, '600')
+        git_commit!
+        raise('Error saving configuration') unless working_directory_clean?
+        # TODO:  Turn delete back on
+        # File.delete(::Pvcglue.cloud.local_file_name)
+      end
+
+      def git_commit!
+        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git add -A && git commit --allow-empty --author="pvc-$PVCGLUE_USER <>" -m "Change configuration"))
+      end
+
+      def working_directory_clean?
+        connection.run_get_stdout!(user_name, '', %Q(cd #{manager_dir} && git status)) =~ /working directory clean/
+      end
+
+      def self.pull_configuration
+        new.pull_configuration
+        # new(Pvcglue.cloud.manager_minion).pull_configuration
+      end
+
+      def pull_configuration
+        # TODO:  Rename to edit_config_start, and create Thor commands to match
+        if connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
+          data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
+        else
+          data = "# Pvcglue manager configuration file\n\n"
+        end
+        file_name = ::Pvcglue.cloud.local_file_name
+        if File.exist?(file_name)
+          backup_file_name = ::Pvcglue.configuration.versioned_filename(file_name)
+          File.rename(file_name, backup_file_name)
+          Pvcglue.logger.info("Existing local configuration file saved to #{backup_file_name}")
+        end
+        File.write(file_name, data)
+        puts "Configuration saved to #{file_name}.  Now edit it and push it back up."
+      end
+
+      def load_secrets
+        connection.read_from_file_if_exists?(user_name, ::Pvcglue::Env.stage_env_file_name)
+      end
+
+      def save_secrets(data)
+        connection.write_to_file(user_name, data, ::Pvcglue::Env.stage_env_file_name, nil, nil, 600)
+        git_commit!
+      end
 
-package 'manager-get-config' do
-  apply do
-    data = run("cat #{::Pvcglue::Manager.manager_file_name}")
-    #puts "*"*80
-    #puts data
-    #puts "*"*80
-    if data.empty?
-      raise "Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}"
-    else
-      ::Pvcglue.cloud.data = TOML.parse(data)
     end
   end
 end
-