pvcglue 0.9.4 → 0.9.5

data/lib/pvcglue/minion.rb

@@ -18,6 +18,7 @@ module Pvcglue
       Pvcglue::Packages::UnattendedUpgrades.apply(minion)
       Pvcglue::Packages::Users.apply(minion)
       Pvcglue::Packages::AuthorizedKeys.apply(minion)
+      Pvcglue::Packages::Slack.apply(minion)
       Pvcglue::Packages::Roles.apply(minion)
 
 
@@ -38,9 +39,16 @@ module Pvcglue
       public_ip.present?
     end
 
+    def get_cloud_provider_options
+      # Merge default and machine options
+      options = machine_options.cloud_provider || ::SafeMash.new
+      options.reverse_merge(default_cloud_provider)
+    end
+
     def pvc_cloud_provider
-      byebug unless machine_options.cloud_provider || default_cloud_provider
-      @pvc_cloud_provider ||= Pvcglue::CloudProviders.init(machine_options.cloud_provider || default_cloud_provider)
+      options = get_cloud_provider_options
+      # raise("Unknown cloud provider '#{get_cloud_provider_name}'") unless get_cloud_provider_name
+      @pvc_cloud_provider ||= Pvcglue::CloudProviders.init(options)
     end
 
     def create!
@@ -70,9 +78,6 @@ module Pvcglue
           ssh_keys: ssh_keys,
           backups: backups,
           ipv6: false,
-          private_networking: true,
-          user_data: '',
-          monitoring: true,
           tags: tags,
           group: group,
       )
@@ -174,7 +179,8 @@ module Pvcglue
         keys += get_github_authorized_keys(user)
         keys.each do |public_key|
           line = %Q(environment="PVCGLUE_USER=#{user.name}" #{public_key})
-          # line = %Q(command="export PVCGLUE_USER=#{id}" #{public_key})
+          # line = %Q(command="export PVCGLUE_USER=#{user.name}" #{public_key})
+          # line = %Q(command="path/to/script",environment="LEVEL=1" command="export PVCGLUE_USER=#{user.name}" #{public_key})
           data << line
         end
       end
@@ -186,5 +192,58 @@ module Pvcglue
     #   self.public_ip = another_minion.public_ip
     #   self.cloud_id = another_minion.cloud_id
     # end
+    
+    def sidekiq_worker_base_name # pvc-sidekiq-project-stage-
+      "pvc-sidekiq-#{remote_user_name}-"
+    end
+
+    def sidekiq_service_file_name(name) # /lib/systemd/system/pvc-sidekiq-project-stage-name.service
+      "#{Pvcglue.cloud.service_directory}#{sidekiq_service_name(name)}#{Pvcglue.cloud.service_extension}"
+    end
+
+    def sidekiq_service_name(name) # pvc-sidekiq-project-stage-name
+      "#{sidekiq_worker_base_name}#{name}"
+    end
+
+
+    def quiet_all_sidekiq_workers_cmd
+      cmd = []
+      return cmd unless stage_options.sidekiq_queues
+      stage_options.sidekiq_queues.each do |name, options|
+        cmd << "sudo systemctl kill -s USR1 --kill-who=main #{sidekiq_service_name(name)}"
+      end
+      cmd
+    end
+    
+    def stop_all_sidekiq_workers_cmd
+      cmd = []
+      return cmd unless stage_options.sidekiq_queues
+      stage_options.sidekiq_queues.each do |name, options|
+        cmd << "sudo systemctl stop #{sidekiq_service_name(name)}"
+      end
+      cmd
+    end
+
+    def start_all_sidekiq_workers_cmd
+      cmd = []
+      return cmd unless stage_options.sidekiq_queues
+      stage_options.sidekiq_queues.each do |name, options|
+        cmd << "sudo systemctl start #{sidekiq_service_name(name)}"
+      end
+      cmd
+    end
+
+    def quiet_all_workers_cmd
+      quiet_all_sidekiq_workers_cmd.join('; ')
+    end
+
+
+    def stop_all_workers_cmd
+      stop_all_sidekiq_workers_cmd.join(' && ')
+    end
+
+    def start_all_workers_cmd
+      start_all_sidekiq_workers_cmd.join(' && ')
+    end
   end
 end

data/lib/pvcglue/packages.rb

@@ -10,6 +10,9 @@ module Pvcglue
 
     attr_accessor :errors
     attr_accessor :options
+    attr_accessor :post_install_max_retry_seconds
+    attr_accessor :post_install_max_retry_count
+    attr_accessor :post_install_retry_delay_seconds
 
     def initialize(minion, options = {})
       @minion = minion
@@ -42,13 +45,13 @@ module Pvcglue
         Pvcglue.logger_package_description = self.class.name
         unless installed?
           install!
-          if post_install_check?
+          if post_install_check_with_retry?
             post_install!
           else
             # TODO:  Better error message
 
             errors << 'Install failed post install check.'
-            Pvcglue.logger.error { full_error_message }
+            Pvcglue.logger.error {full_error_message}
             return false
           end
         end
@@ -71,6 +74,27 @@ module Pvcglue
       installed?
     end
 
+    def post_install_retry(max_tries, delay_seconds = 0, max_seconds = 30)
+      self.post_install_max_retry_seconds = max_seconds
+      self.post_install_retry_delay_seconds = delay_seconds
+      self.post_install_max_retry_count = max_tries
+    end
+
+    def post_install_check_with_retry?
+      self.post_install_max_retry_seconds = 0.0
+      self.post_install_retry_delay_seconds = 0.0
+      self.post_install_max_retry_count = 1
+      started_at = Time.now.utc.to_f
+      tries = 0
+      begin
+        return true if post_install_check?
+        tries += 1
+        Pvcglue.logger.debug('Failed post install check, retrying...')
+        sleep(post_install_retry_delay_seconds)
+      end until tries >= post_install_max_retry_count || Time.now.utc.to_f - started_at > post_install_max_retry_seconds
+      false
+    end
+
     def post_install!
 
     end
@@ -113,12 +137,20 @@ module Pvcglue
       end
     end
 
-    def get_minion_state(key)
+    def get_minion_state(key = nil)
+      key = get_minion_state_key(key)
       get_minion_state_data
       connection.minion_state_data[key]
     end
 
-    def set_minion_state(key, value)
+    def get_minion_state_key(key)
+      # TODO:  Use a versioned state, so that when the Pvcglue version is updated, it will rebuild all
+    key || self.class.name.downcase.gsub(':', '_').to_sym
+    end
+
+    def set_minion_state(key = nil, value = nil)
+      value ||= Time.now.utc
+      key = get_minion_state_key(key)
       get_minion_state_data
       connection.minion_state_data[key] = value
       connection.write_to_file(:root, TOML.dump(connection.minion_state_data), minion_state_file_name)

data/lib/pvcglue/packages/apt.rb

@@ -13,6 +13,7 @@ module Pvcglue
         nginx
         nginx-extras
         nodejs
+        redis-tools
       ]
       PACKAGES = {
           common: %w[
@@ -37,6 +38,9 @@ module Pvcglue
             postgresql-contrib-9.6
             libpq-dev
           ],
+          redis: %w[
+            redis-server
+          ],
           mc: %w[
           ],
       }

data/lib/pvcglue/packages/apt_repos.rb

@@ -8,6 +8,11 @@ module Pvcglue
         has_roles? %w(lb web)
       end
 
+      def redis_needed?
+        # has_roles? %w(redis)
+        has_roles? %w(redis web worker) # web worker will just have package `redis-tools`
+      end
+
       def node_js_needed?
         has_roles? %w(web worker)
       end
@@ -58,6 +63,16 @@ module Pvcglue
           end
         end
 
+        if redis_needed?
+          docs.set_item(
+            heading: 'Redis',
+            body: 'Install the latest stable version.  The current Ubuntu version is apparently behind on security updates.',
+            reference: 'https://www.linode.com/docs/databases/redis/deploy-redis-on-ubuntu-or-debian'
+          ) do
+            connection.run!(:root, '', 'add-apt-repository "ppa:chris-lea/redis-server"')
+          end
+        end
+
         if node_js_needed?
           # Reference:  http://tecadmin.net/install-latest-nodejs-npm-on-ubuntu/
           connection.run!(:root, '', 'apt-get install -y apt-transport-https ca-certificates python-software-properties lsb-release')

data/lib/pvcglue/packages/authorized_keys.rb

@@ -10,30 +10,57 @@ module Pvcglue
       end
 
       def install!
-        # TODO:  Safety check to see if user is locking himself out.  :)
-        if manager_first_bootstrap?
-          # TODO:  work out system for pvc-manager access
-          data = [`cat ~/.ssh/id_rsa.pub`.strip]
-        else
-          data = minion.get_root_authorized_keys_data
-          if data.count == 0
-            raise('No authorized keys found for root users!')
+        docs.set_item(
+          heading: 'Authorized Users',
+          body: 'Configures sshd and the authorized_keys files.',
+          notes: [
+            ''
+          ],
+          cheatsheet: [
+            '',
+          ],
+          references: [
+            'https://serverfault.com/questions/256098/authorized-keys-environment-variables-not-setting-environment-variables',
+            'https://serverfault.com/questions/527638/security-risks-of-permituserenvironment-in-ssh',
+            '',
+            'https://www.digitalocean.com/community/tutorials/how-to-read-and-set-environmental-and-shell-variables-on-a-linux-vps',
+            'https://binblog.info/2008/10/20/openssh-going-flexible-with-forced-commands/',
+            'https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.2.0/com.ibm.zos.v2r2.foto100/authkeyf.htm',
+            'https://en.wikibooks.org/wiki/OpenSSH/Client_Configuration_Files#environment.3D.22NAME.3Dvalue.22',
+            'http://man.openbsd.org/sshd_config.5',
+            'https://developer.rackspace.com/blog/speeding-up-ssh-session-creation/',
+          ]
+        ) do
+          # sshd configuration
+          connection.write_to_file_from_template(:root, 'sshd_config.erb', '/etc/ssh/sshd_config')
+          connection.run!(:root, '', 'systemctl restart sshd')
+
+          # authorized_keys
+          # TODO:  Safety check to see if user is locking himself out.  :)
+          if manager_first_bootstrap?
+            # TODO:  work out system for pvc-manager access
+            data = [`cat ~/.ssh/id_rsa.pub`.strip]
+          else
+            data = minion.get_root_authorized_keys_data
+            if data.count == 0
+              raise('No authorized keys found for root users!')
+            end
           end
-        end
-        connection.write_to_file(:root, data.join("\n"), '/root/.ssh/authorized_keys')
+          connection.write_to_file(:root, data.join("\n"), '/root/.ssh/authorized_keys')
 
-        connection.mkdir_p(:root, "/home/#{user_name}/.ssh", user_name, user_name, '0700')
+          connection.mkdir_p(:root, "/home/#{user_name}/.ssh", user_name, user_name, '0700')
 
-        if manager_first_bootstrap?
-          data = [`cat ~/.ssh/id_rsa.pub`.strip]
-        else
-          data = minion.get_users_authorized_keys_data
-          if data.count == 0
-            raise('No authorized keys found for users!')
-            # TODO:  work out system for pvc-manager access
+          if manager_first_bootstrap?
+            data = [`cat ~/.ssh/id_rsa.pub`.strip]
+          else
+            data = minion.get_users_authorized_keys_data
+            if data.count == 0
+              raise('No authorized keys found for users!')
+              # TODO:  work out system for pvc-manager access
+            end
           end
+          connection.write_to_file(:root, data.join("\n"), user_authorized_keys_file_name, user_name, user_name, '0644')
         end
-        connection.write_to_file(:root, data.join("\n"), user_authorized_keys_file_name, user_name, user_name, '0644')
       end
 
       def user_authorized_keys_file_name

data/lib/pvcglue/packages/firewall.rb

@@ -4,8 +4,7 @@ module Pvcglue
       # Reference:  http://manpages.ubuntu.com/manpages/xenial/en/man8/ufw-framework.8.html
       # Examples:  https://help.ubuntu.com/community/UFW
       def installed?
-        result = connection.run_get_stdout!(:root, '', 'ufw status verbose')
-        result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
+        get_minion_state
       end
 
       def install!
@@ -26,8 +25,13 @@ module Pvcglue
           end
         end
 
+        set_minion_state
       end
 
+      def post_install_check?
+        result = connection.run_get_stdout!(:root, '', 'ufw status verbose')
+        result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
+      end
     end
   end
 end

data/lib/pvcglue/packages/manager.rb

@@ -82,7 +82,7 @@ module Pvcglue
         raise('Not supported for local manager') if Pvcglue.command_line_options[:cloud_manager_override]
 
         test_data = File.read(::Pvcglue.cloud.local_file_name)
-        TOML.parse(test_data) # At least make sure it's valid TOML
+        ::SafeMash.new(TOML.parse(test_data)) # At least make sure it's valid TOML and that we can load it in
 
         # TODO:  More in-depth validations
 
@@ -94,7 +94,9 @@ module Pvcglue
       end
 
       def git_commit!
-        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git add -A && git commit --allow-empty --author="pvc-$PVCGLUE_USER <>" -m "Change configuration"))
+        # connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git add -A && git commit --allow-empty --author="pvc-$PVCGLUE_USER <>" -m "Change configuration")) # Will not work as $PVCGLUE_USER is in the local context here
+        connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git add -A && git commit --allow-empty --author="pvc-\\$PVCGLUE_USER <>" -m "Change configuration")) # It's all about context!
+        # connection.ssh!(user_name, '', %Q(cd #{manager_dir} && git add -A && git commit --allow-empty --author="pvc-\\`printenv PVCGLUE_USER\\` <>" -m "Change configuration")) # Another way that works...for future reference
       end
 
       def working_directory_clean?
@@ -107,6 +109,7 @@ module Pvcglue
       end
 
       def pull_configuration
+        # ssh REMOTE "sh -c \"(nohup sleep 30; touch nohup-exit) > /dev/null &\""
         # TODO:  Rename to edit_config_start, and create Thor commands to match
         if connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
           data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)

data/lib/pvcglue/packages/postgresql.rb

@@ -7,9 +7,12 @@ module Pvcglue
       end
 
       def install!
+        # TODO:  Adjust vm.overcommit_memory?
+        # See https://www.postgresql.org/docs/9.3/static/kernel-resources.html#LINUX-MEMORY-OVERCOMMIT
+        # http://serverfault.com/questions/755772/redis-and-postgresql-on-same-machine-vm-overcommit-memory
         Pvcglue::Env.initialize_stage_env
-        connection.write_to_file_from_template(:root, 'postgresql.conf.erb', '/etc/postgresql/9.6/main/postgresql.conf', 'postgres', 'postgres', '0644')
-        connection.write_to_file_from_template(:root, 'pg_hba.conf.erb', '/etc/postgresql/9.6/main/pg_hba.conf', 'postgres', 'postgres', '0644')
+        connection.write_to_file_from_template(:root, 'postgresql.conf.erb', '/etc/postgresql/9.6/main/postgresql.conf', nil, 'postgres', 'postgres', '0644')
+        connection.write_to_file_from_template(:root, 'pg_hba.conf.erb', '/etc/postgresql/9.6/main/pg_hba.conf', nil, 'postgres', 'postgres', '0644')
 
         connection.run_get_stdout(:root, '', 'service postgresql restart')
         unless $?.exitstatus == 0

data/lib/pvcglue/packages/redis.rb

@@ -0,0 +1,59 @@
+module Pvcglue
+  class Packages
+    class Redis < Pvcglue::Packages
+
+      def installed?
+        get_minion_state
+      end
+
+      def install!
+        docs.set_item(
+          heading: 'Redis',
+          body: 'Set the recommended persistence and memory settings.',
+          notes: [
+            'If multiple applications use the same virtual machine, set a unique database number (0-15) for each one `REDIS_URL=redis://<redis_private_ip>:6379/<number>`'
+          ],
+          cheatsheet: [
+            'System Level Status(run on Redis server):  sudo systemctl status redis',
+            'Redis Info (run on Redis server):  redis-cli info',
+            'Redis Status (run on Redis server):  redis-cli --stat',
+            'Monitor Redis Commands (run on Redis server):  redis-cli monitor',
+            'Latency (run on Redis server):  redis-cli --latency',
+            'Intrinsic Latency (run on Redis server):  redis-cli --intrinsic-latency 10',
+            'Ping Redis (run from web or worker):  redis-cli -h xxx.xxx.xxx.xxx ping',
+            'Redis Status (run from web or worker):  redis-cli -h xxx.xxx.xxx.xxx --stat',
+            'Latency (run from web or worker):  redis-cli -h xxx.xxx.xxx.xxx --latency',
+            'Latency Dist (run from web or worker):  redis-cli -h xxx.xxx.xxx.xxx --latency-dist',
+          ],
+          references: [
+            '[CLI Reference]https://redis.io/topics/rediscli',
+            'https://www.linode.com/docs/databases/redis/deploy-redis-on-ubuntu-or-debian',
+            'https://redis.io/topics/faq',
+            'http://serverfault.com/questions/485798/cent-os-how-do-i-turn-off-or-reduce-memory-overcommitment-and-is-it-safe-to-do',
+            'https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-redis-on-ubuntu-16-04',
+            'http://stackoverflow.com/questions/21795340/linux-install-redis-cli-only',
+          ]
+        ) do
+          # Persistence
+          connection.write_to_file_from_template(:root, 'redis.conf.erb', '/etc/redis/redis.conf')
+          connection.run!(:root, '', 'service redis-server restart')
+
+          # Memory management
+          connection.run!(:root, '', 'sysctl vm.overcommit_memory=1') # Not sure if this is necessary if doing `sysctl -p`
+          connection.write_to_file_from_template(:root, 'sysctl.conf.erb', '/etc/sysctl.conf')
+          connection.run!(:root, '', 'sysctl -p')
+        end
+
+        set_minion_state
+      end
+
+      def post_install_check?
+        post_install_retry(20,0.5)
+        redis_info = connection.run_get_stdout!(user_name, '', 'redis-cli info')
+        no_eviction = redis_info =~ /maxmemory_policy:noeviction/
+        aof_enabled = redis_info =~ /aof_enabled:1/
+        get_minion_state && no_eviction && aof_enabled
+      end
+    end
+  end
+end

data/lib/pvcglue/packages/rvm.rb

@@ -13,7 +13,7 @@ module Pvcglue
         # Do it again, the first time only sets things up, and does not import the keys
         connection.run!(user_name, '', '\curl -sSL https://get.rvm.io | bash -s stable --with-default-gems=bundler')
 
-        # TODO: set autolibs mode so there are not installed automatically, as the user won't have sudo permissions later.
+        # TODO: set autolibs mode so they are not installed automatically, as the user won't have sudo permissions later.
         # OR create a 'install' user that can sudo...might be easier...
         # Installing required packages: libreadline6-dev, libyaml-dev, libsqlite3-dev, sqlite3, autoconf, libgmp-dev, libgdbm-dev, libncurses5-dev, automake, libtool, bison, pkg-config, libffi-dev...............
 

data/lib/pvcglue/packages/secrets.rb

@@ -7,8 +7,9 @@ module Pvcglue
 
       def install!
         Pvcglue::Env.initialize_stage_env
-        connection.write_to_file_from_template(user_name, 'web.env.erb', Pvcglue.cloud.env_file_name, nil, nil, '0640') # TODO:  Double check permissions
+        connection.write_to_file_from_template(user_name, 'web.env.erb', Pvcglue.cloud.env_file_name, nil, nil, nil, '0640') # TODO:  Double check permissions
         restart_web_app!
+        restart_workers!
       end
 
       def post_install_check?
@@ -21,6 +22,11 @@ module Pvcglue
         end
       end
 
+      def restart_workers!
+        puts ('*'*800).red
+        puts 'Workers not restarted!!!'.yellow
+      end
+
       def self.load_for_stage
         data = Pvcglue::Packages::Manager.new.load_secrets
         data = '' if data.nil?