pvcglue 0.9.3 → 0.9.4

data/lib/pvcglue/configuration.rb

@@ -60,6 +60,7 @@ module Pvcglue
       end
 
       def configure_manager
+        byebug
         manager = ask('What is the IP address or host name of the manager?')
         default = !no?('Will this be the default manager? (Y/n)')
         file_name = default ? user_file_name : project_file_name
@@ -145,6 +146,10 @@ module Pvcglue
         File.join(tmp_dir, 'pvcglue')
       end
 
+      def template_override_dir
+        File.join(application_dir, 'config', 'pvcglue', 'templates')
+      end
+
       def cloud_cache_file_name
         # Just in case the Rails project hasn't yet been run, make sure the tmp
         # dir exists.

data/lib/pvcglue/connection.rb

@@ -75,6 +75,7 @@ module Pvcglue
 
     def system_command?(cmd)
       Pvcglue.logger.debug { cmd }
+      Pvcglue.docs.log_cmd(cmd)
       system(cmd)
       Pvcglue.logger.debug { "exit_code=#{$?.exitstatus}" }
       $?
@@ -170,6 +171,7 @@ module Pvcglue
     end
 
     def write_to_file(user, data, file, owner = nil, group = nil, permissions = nil)
+      Pvcglue.docs.log_file_write(user: user, style: :shell, data: data, file: file, owner: owner, group: group, permissions: permissions)
       tmp_file = Tempfile.new('pvc')
       begin
         tmp_file.write(data)

data/lib/pvcglue/docs.rb

@@ -0,0 +1,97 @@
+module Pvcglue
+  class Docs
+    ROLLS = {
+      lb: 'Load Balancer'
+    }
+
+    def initialize(enabled)
+      @enabled = enabled
+      @data = []
+    end
+
+    attr_accessor :enabled, :data, :current_minion, :collecting
+
+    def add(s)
+      return unless collecting
+      data << s
+    end
+
+    def add_paragraph(text)
+      add('')
+      add(text)
+      add('')
+    end
+
+    def add_annotation(text)
+      add('')
+      add("> #{text}")
+      add('')
+    end
+
+
+    def level_1_roles(minion)
+      return unless enabled
+      self.current_minion = minion
+      if minion
+        self.collecting = true
+        level_1(minion.roles.map { |role| ROLLS[role.to_sym] || role }.join(', '))
+      else
+        self.collecting = false
+      end
+    end
+
+    def level_1(heading)
+      return unless enabled
+      add_header(1, heading)
+    end
+
+    def level_2(heading)
+      return unless enabled
+      add_header(2, heading)
+    end
+
+    def add_header(level, heading)
+      return unless enabled
+      add('')
+      add("#{'#'*level} #{heading}")
+    end
+
+    def set_item(options)
+      return unless enabled
+      options = ::SafeMash.new(options)
+      add_header(3, options.heading)
+      add('----')
+      yield
+      add_paragraph(options.body) if options.body && options.body.present?
+      if options.reference && options.reference.present?
+        add('')
+        add("See also:  [#{options.reference}](#{options.reference})")
+      end
+    end
+
+    def log_file_write(options)
+      return unless enabled
+      options = ::SafeMash.new(options)
+      add_annotation("Write data to `#{options.file}`")
+      add_block(options)
+    end
+
+    def log_cmd(cmd)
+      return unless enabled
+      # add_block(data: cmd, style: 'shell')
+      add("> `> #{cmd}`<br />")
+    end
+
+    def add_block(options)
+      options = ::SafeMash.new(options)
+      add("```#{options.style}")
+      add(options.data)
+      add('```')
+    end
+
+    def done
+      return unless enabled
+      File.write('/home/andrew/projects/slate-pvc/source/includes/_logs.md', data.join("\n"))
+    end
+  end
+end

data/lib/pvcglue/manager.rb

@@ -191,7 +191,6 @@ module Pvcglue
 
     def self.push_configuration
       Pvcglue::Packages::Manager.push_configuration
-      # Pvcglue::Packages.apply('manager-push'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
       # clear_cloud_data_cache
     end
 

data/lib/pvcglue/minion.rb

@@ -3,10 +3,11 @@ module Pvcglue
     include Hashie::Extensions::Mash::SafeAssignment
 
     def build!
-      # puts '*'*175
+      minion = self # for readability
+
       Pvcglue.logger.info('BUILD') { "Building #{machine_name}" }
+      Pvcglue.docs.level_1_roles(minion)
 
-      minion = self # for readability
       Pvcglue::Packages::SshKeyCheck.apply(minion)
       Pvcglue::Packages::AptRepos.apply(minion)
       Pvcglue::Packages::AptUpdate.apply(minion)
@@ -37,40 +38,47 @@ module Pvcglue
       public_ip.present?
     end
 
+    def pvc_cloud_provider
+      byebug unless machine_options.cloud_provider || default_cloud_provider
+      @pvc_cloud_provider ||= Pvcglue::CloudProviders.init(machine_options.cloud_provider || default_cloud_provider)
+    end
+
     def create!
-      raise(Thor::Error, "#{cloud_provider.name} unknown") unless cloud_provider.name == 'digital-ocean'
-      Pvcglue.logger.warn("Provisioning a machine for #{machine_name} on #{cloud_provider.name}...")
+      Pvcglue.logger.warn("Provisioning a machine for #{machine_name} on #{pvc_cloud_provider.name}...")
 
       # TODO:  Tags.  production, staging, load-balancer, web, worker, database, postgress, cache, memcache...
+
       name = machine_name
-      size = capacity || cloud_provider.default_capacity
-      image = cloud_provider.image
-      region = cloud_provider.region
-      # ap cloud_provider.initial_users
-      # ap cloud_provider
-      ssh_keys = cloud_provider.initial_users.map { |description, ssh_key| ssh_key }
-      backups = cloud_provider.backups.nil? ? true : cloud_provider.backups # default to true -- safety first!
-      tags = cloud_provider.tags
-
-      # client.droplets.all.each do |droplet|
-      #   ap droplet
-      # end
-      droplet = DropletKit::Droplet.new(
+      capacity = pvc_cloud_provider.options.capacity
+      image = pvc_cloud_provider.options.image
+      region = pvc_cloud_provider.options.region
+      if pvc_cloud_provider.options.initial_users
+        ssh_keys = pvc_cloud_provider.options.initial_users.map { |description, ssh_key| ssh_key }
+      else
+        ssh_keys = []
+      end
+      # backups = cloud_provider.backups.nil? ? true : cloud_provider.backups # default to true -- safety first!
+      backups = pvc_cloud_provider.options.backups.nil? ? true : machine_options.cloud_provider.backups # default to true -- safety first!
+      tags = pvc_cloud_provider.options.tags
+      group = pvc_cloud_provider.options.group
+
+      options = ::SafeMash.new(
           name: name,
           region: region,
           image: image,
-          size: size,
+          capacity: capacity,
           ssh_keys: ssh_keys,
           backups: backups,
           ipv6: false,
           private_networking: true,
           user_data: '',
           monitoring: true,
-          tags: tags
+          tags: tags,
+          group: group,
       )
-      droplet = Pvcglue::DigitalOcean.client.droplets.create(droplet)
-      self.droplet = droplet
-      Pvcglue.logger.debug("Droplet ID:  #{droplet.id}")
+
+      machine = pvc_cloud_provider.create(options)
+      self.machine = machine
       true
     end
 

data/lib/pvcglue/packages.rb

@@ -17,6 +17,10 @@ module Pvcglue
       @errors = []
     end
 
+    def docs
+      Pvcglue.docs
+    end
+
     def errors?
       errors.size > 0
     end

data/lib/pvcglue/packages/apt_repos.rb

@@ -21,13 +21,41 @@ module Pvcglue
       end
 
       def install!
+        docs.level_2('Repositories')
+
+        # TODO: Make this a package that checks for the existence of software-properties-common
+        #echo 'Acquire::ForceIPv4 "true";' | tee /etc/apt/apt.conf.d/99force-ipv4
+        docs.set_item(
+          heading: 'Force use of IPv4',
+          body: 'Needed for Linode as there were intermittent problems connecting to their repositories over IPv6 (February 2017)'
+        ) do
+          connection.write_to_file(:root, 'Acquire::ForceIPv4 "true";', '/etc/apt/apt.conf.d/99force-ipv4')
+        end
+
+
+        docs.set_item(
+          heading: 'Update Repositories'
+        ) do
+          connection.run!(:root, '', 'apt update -y')
+        end
+        docs.set_item(
+          heading: 'Install Requirements',
+          body: 'Install the requirements for adding more repositories'
+        ) do
+          connection.run!(:root, '', 'apt install -y software-properties-common python-software-properties')
+        end
         # These could be refactored into packages.  :)
 
         if nginx_needed?
-          # Reference:  https://www.phusionpassenger.com/library/install/nginx/install/oss/xenial/
-          connection.run!(:root, '', 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7')
-          connection.run!(:root, '', 'apt-get install -y apt-transport-https ca-certificates')
-          connection.write_to_file(:root, PASSENGER_SOURCES_LIST_DATA, PASSENGER_SOURCES_LIST_FILENAME)
+          docs.set_item(
+            heading: 'Nginx',
+            body: 'Install the latest version using the Phusion repos.',
+            reference: 'https://www.phusionpassenger.com/library/install/nginx/install/oss/xenial/'
+          ) do
+            connection.run!(:root, '', 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7')
+            connection.run!(:root, '', 'apt-get install -y apt-transport-https ca-certificates')
+            connection.write_to_file(:root, PASSENGER_SOURCES_LIST_DATA, PASSENGER_SOURCES_LIST_FILENAME)
+          end
         end
 
         if node_js_needed?
@@ -40,6 +68,7 @@ module Pvcglue
           connection.run!(:root, '', 'add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main"')
           connection.run!(:root, '', 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -')
         end
+        # byebug # docs resume here
 
         set_minion_state(:apt_repos_updated_at, Time.now.utc)
       end

data/lib/pvcglue/packages/authorized_keys.rb

@@ -11,22 +11,38 @@ module Pvcglue
 
       def install!
         # TODO:  Safety check to see if user is locking himself out.  :)
-        data = minion.get_root_authorized_keys_data
-        if data.count == 0
-          raise('No authorized keys found for root users!')
+        if manager_first_bootstrap?
           # TODO:  work out system for pvc-manager access
           data = [`cat ~/.ssh/id_rsa.pub`.strip]
+        else
+          data = minion.get_root_authorized_keys_data
+          if data.count == 0
+            raise('No authorized keys found for root users!')
+          end
         end
         connection.write_to_file(:root, data.join("\n"), '/root/.ssh/authorized_keys')
 
         connection.mkdir_p(:root, "/home/#{user_name}/.ssh", user_name, user_name, '0700')
-        data = minion.get_users_authorized_keys_data
-        if data.count == 0
-          raise('No authorized keys found for users!')
-          # TODO:  work out system for pvc-manager access
+
+        if manager_first_bootstrap?
           data = [`cat ~/.ssh/id_rsa.pub`.strip]
+        else
+          data = minion.get_users_authorized_keys_data
+          if data.count == 0
+            raise('No authorized keys found for users!')
+            # TODO:  work out system for pvc-manager access
+          end
         end
-        connection.write_to_file(:root, data.join("\n"), "/home/#{user_name}/.ssh/authorized_keys", user_name, user_name, '0644')
+        connection.write_to_file(:root, data.join("\n"), user_authorized_keys_file_name, user_name, user_name, '0644')
+      end
+
+      def user_authorized_keys_file_name
+        "/home/#{user_name}/.ssh/authorized_keys"
+      end
+
+      def manager_first_bootstrap?
+        return false unless has_role?(:manager)
+        @manager_first_bootstrap ||= !Pvcglue::Packages::Manager.configuration_exists?
       end
     end
   end

data/lib/pvcglue/packages/firewall.rb

@@ -21,6 +21,7 @@ module Pvcglue
         unless has_role?(:manager)
           minion.cloud.minions.each do |other_minion_name, other_minion|
             next if other_minion_name == minion.machine_name
+            next unless other_minion.provisioned?
             connection.run!(:root, '', "ufw allow from #{other_minion.private_ip}")
           end
         end

data/lib/pvcglue/packages/load_balancer.rb

@@ -6,6 +6,8 @@ module Pvcglue
       end
 
       def install!
+        sync_maintenance_files
+
         if Pvcglue.cloud.ssl_mode == :acme && !connection.file_exists?(:root, Pvcglue.cloud.nginx_ssl_crt_file_name)
           # Don't include the SSL stuff in the Nginx config until we have a cert,
           # but Nginx has to be configured to get the cert from Let's Encrypt
@@ -24,7 +26,6 @@ module Pvcglue
           nginx_restart!
         end
 
-        sync_maintenance_files
       end
 
       def nginx_restart!
@@ -56,10 +57,12 @@ module Pvcglue
       end
 
       def sync_maintenance_files
+        Pvcglue.logger.debug { 'Synchronizing maintenance mode files' }
         source_dir = Pvcglue.configuration.app_maintenance_files_dir
         dest_dir = Pvcglue.cloud.maintenance_files_dir
         maintenance_file_name = File.join(source_dir, 'maintenance.html')
         unless File.exists?(maintenance_file_name)
+          Pvcglue.logger.debug { 'Creating default maintenance mode files' }
           # TODO:  Make this use a template
           `mkdir -p #{source_dir}`
           File.write(maintenance_file_name, '-Maintenance Mode-')

data/lib/pvcglue/packages/manager.rb

@@ -48,6 +48,14 @@ module Pvcglue
         new.get_configuration
       end
 
+      def self.configuration_exists?
+        new.configuration_exists?
+      end
+
+      def configuration_exists?
+        connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
+      end
+
       def get_configuration
         # if connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
         #   data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
@@ -56,8 +64,12 @@ module Pvcglue
         #   raise("Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}")
         # end
         data = '' # to use `data` in block
-        Pvcglue.filter_verbose do
-          data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
+        if Pvcglue.command_line_options[:cloud_manager_override]
+          data = File.read(Pvcglue.command_line_options[:cloud_manager_override])
+        else
+          Pvcglue.filter_verbose do
+            data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
+          end
         end
         ::Pvcglue.cloud.data = TOML.parse(data)
       end
@@ -67,6 +79,13 @@ module Pvcglue
       end
 
       def push_configuration
+        raise('Not supported for local manager') if Pvcglue.command_line_options[:cloud_manager_override]
+
+        test_data = File.read(::Pvcglue.cloud.local_file_name)
+        TOML.parse(test_data) # At least make sure it's valid TOML
+
+        # TODO:  More in-depth validations
+
         connection.upload_file(user_name, ::Pvcglue.cloud.local_file_name, ::Pvcglue::Manager.manager_file_name, nil, nil, '600')
         git_commit!
         raise('Error saving configuration') unless working_directory_clean?
@@ -92,8 +111,10 @@ module Pvcglue
         if connection.file_exists?(user_name, ::Pvcglue::Manager.manager_file_name)
           data = connection.read_from_file(user_name, ::Pvcglue::Manager.manager_file_name)
         else
-          data = "# Pvcglue manager configuration file\n\n"
+          template = Tilt.new(Pvcglue.template_file_name('pvc_manager.toml.erb'))
+          data = template.render(self, minion: minion)
         end
+
         file_name = ::Pvcglue.cloud.local_file_name
         if File.exist?(file_name)
           backup_file_name = ::Pvcglue.configuration.versioned_filename(file_name)

data/lib/pvcglue/packages/ssh_key_check.rb

@@ -2,8 +2,14 @@ module Pvcglue
   class Packages
     class SshKeyCheck < Pvcglue::Packages
       def installed?
+        docs.level_2('Status')
         # This has the side effect of adding the server to known_hosts file, to prevent needing interactive prompt
-        connection.ssh_retry_wait(:root, '-o strictHostKeyChecking=no', 'echo', 30, 1)
+        docs.set_item(
+          heading: 'Verify Connection',
+          body: 'Connect to the machine and wait until it''s ready (with automatic retry).  Also add the machine to the `known_hosts` file, to prevent an interactive prompt'
+        ) do
+          connection.ssh_retry_wait(:root, '-o strictHostKeyChecking=no', 'echo', 30, 1)
+        end
         true
       end
     end

data/lib/pvcglue/packages/ssl.rb

@@ -21,9 +21,19 @@ module Pvcglue
         force_option = Pvcglue.command_line_options[:force_cert] ? '--force ' : ''
         debug_option = Pvcglue.logger.level == 0 ? '--debug ' : ''
 
-        unless Net::HTTP.get(first_domain, '/.well-known/acme-challenge/test.html') =~ /shiny/
-          Pvcglue.logger.error("Unable to connect to #{first_domain} at #{minion.public_ip}")
-          raise(Thor::Error, 'Please fix and then restart.')
+        begin
+
+          # Test with http://www.example.com/.well-known/acme-challenge/test.html
+          base_name = "test-#{SecureRandom.hex}.html"
+          verification_file_name = File.join(Pvcglue.cloud.letsencrypt_full, base_name)
+          connection.write_to_file(:root, "Everything's shiny, Cap'n. Not to fret.", verification_file_name, 'root', 'www-data', '660')
+
+          unless Net::HTTP.get(first_domain, "/.well-known/acme-challenge/#{base_name}") =~ /shiny/
+            Pvcglue.logger.error("Unable to connect to #{first_domain} at #{minion.public_ip}")
+            raise(Thor::Error, 'Please fix and then restart.')
+          end
+        ensure
+          # TODO:  Delete verification file
         end
 
         result = connection.ssh?(:root, '', "/root/.acme.sh/acme.sh #{debug_option}#{staging_option}#{force_option}--issue #{domain_options} -w #{Pvcglue.cloud.letsencrypt_root}")