pvcglue 0.1.39 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 463623f85adf15ee50e8f2c2425482c2c751f30b
-  data.tar.gz: fa759839920a81f2357b43311ead1622ff72ea5c
+  metadata.gz: 29094a42e1588de62c31f9259be0bc1d0f9d4c91
+  data.tar.gz: cc6c3e188893749c8082f821c751992168d1e02e
 SHA512:
-  metadata.gz: c087b1c4ada832eae9d5cd1b4c4ca004b60269e6b08c8a917c7e1a9aa9996ef7020b64f8153edc38d8ea56061fbe707982f990970332bf5bc8f27b26f10c809b
-  data.tar.gz: 103ab848521a649ccb76a42336d73c1ad71f3eaefeb2f2d5db0b0819bf0c45d655ff4ce0ba13595af9d3e8b7e53e961587dbaa186845a96ae3877f8b370c5e1e
+  metadata.gz: 7a87fabdd8819a42e894eaa53253f76585fadeb29edf55989e9e34df16ce720caed49205a425bdda95256b122b5a3cc8624db86cee24906cc95a0eb9950ded82
+  data.tar.gz: 2add0cab736ec4a2c00aef8e29717a7f3c603f4764f39ef2a3d40353557ec26675cb0bfef8aa722083666c8916d43725b0ea7a62517da824224e7e93b6a547d8

data/.ruby-gemset

@@ -0,0 +1 @@
+pvcglue

data/.ruby-version

@@ -0,0 +1 @@
+2.4.0

data/README.md

@@ -14,14 +14,23 @@ An opinionated cloud application manager for Rails applications using your own s
 
 Currently supported stack:
 
-  * Ubuntu 12.04 (with plans for supporting 14.04)
+  * SSL support:  none, manual and automatic with Let's Encrypt
+  * Ubuntu 16.04 LTS
+  * Provision servers automatically on Digital Ocean (and Linode)
+  * No need to install anything on servers first (you just need SSH access)
   * Ruby >= 1.9 (multiple versions supported on same server)
   * Rails >= 3.2
   * RVM
-  * Postgresql 9.1
+  * Postgresql 9.6
   * Nginx
-  * Passenger > 4.0
+  * Passenger > 5.0
   * Memcached
+  * Redis
+
+Workers:
+
+  * Delayed Job
+  * Rescue
 
 # This is a work in progress
 

data/bin/pvc

@@ -9,6 +9,8 @@ if ARGV.count >= 2 && %w[local vmtest test alpha beta gamma delta preview produc
   ARGV[0], ARGV[1] = ARGV[1], "--stage=#{ARGV[0]}"
 end
 
-# puts ARGV.inspect
+Pvcglue.logger.info('Starting up...')
+Pvcglue.logger.debug { ARGV.inspect }
 # puts Benchmark.measure { Pvcglue::CLI.start }
-puts "----- Done #{Benchmark.measure { Pvcglue::CLI.start }}"
+# pvc_puts.warn("----- Done #{Benchmark.measure { Pvcglue::CLI.start }}")
+Pvcglue.logger.info("----- Done #{Benchmark.measure { Pvcglue::CLI.start }}")

data/lib/pvcglue.rb

@@ -1,29 +1,114 @@
-require "pvcglue/version"
-require "thor"
-require "pvcglue/configuration"
-require "pvcglue/manager"
-require "pvcglue/cloud"
-require "pvcglue/packages"
-require "pvcglue/bootstrap"
-require "pvcglue/nodes"
-require "pvcglue/env"
-require "pvcglue/deploy"
-require "pvcglue/capistrano"
-require "pvcglue/ssl"
-require "pvcglue/db"
-require "pvcglue/toml_pvc_dumper"
-require "pvcglue/local"
-require "pvcglue/monit"
-require "pvcglue/pvcify"
-require "tilt"
+require 'pvcglue/version'
+require 'thor'
+require 'pvcglue/configuration'
+require 'pvcglue/manager'
+require 'pvcglue/cloud'
+require 'pvcglue/packages'
+Dir[File.dirname(__FILE__) + '/pvcglue/packages/*.rb'].each { |file| require file }
+require 'pvcglue/bootstrap'
+require 'pvcglue/nodes'
+require 'pvcglue/stack'
+require 'pvcglue/env'
+require 'pvcglue/deploy'
+require 'pvcglue/capistrano'
+require 'pvcglue/ssl'
+require 'pvcglue/db'
+require 'pvcglue/toml_pvc_dumper'
+require 'pvcglue/local'
+require 'pvcglue/monit'
+require 'pvcglue/pvcify'
+require 'tilt'
+require 'awesome_print'
+require 'hashie'
+require 'pvcglue/custom_hashie'
+require 'pvcglue/minion'
+require 'droplet_kit'
+require 'pvcglue/digital_ocean'
+require 'logger'
+require 'pvcglue/connection'
+require 'paint'
+require 'pry'
 
 # puts File.join(File.dirname(__FILE__), 'pvcglue', 'packages', '*.rb')
+# pvc manager bootstrap --cloud_manager_override=local_cloud.pvcglue.toml --save_before_upload=save --verbose
 
 module Pvcglue
   mattr_accessor :command_line_options do
     {}
   end
 
+  mattr_accessor :logger do
+
+    logger = Logger.new(STDOUT)
+    # logger.level = Logger::INFO # DEBUG < INFO < WARN < ERROR < FATAL < UNKNOWN
+    logger.level = Logger::DEBUG # DEBUG < INFO < WARN < ERROR < FATAL < UNKNOWN
+    # logger.warn('Starting up...')
+    logger.formatter = proc do |severity, datetime, progname, msg|
+      minion_name = Pvcglue.logger_current_minion.try(:machine_name)
+      minion_name = "/#{minion_name}" if minion_name
+      description = Pvcglue.logger_package_description
+      if description
+        description = description.split('::').last || description
+        description = "/#{description.downcase}"
+      end
+      foreground = nil
+      # background = 'black'
+      background = nil
+      case severity[0..0]
+        when 'E'
+          foreground = 'red'
+        when 'W'
+          # foreground = 'black'
+          # background = 'yellow'
+          foreground = 'yellow'
+        when 'I'
+          foreground = 'purple'
+        when 'D'
+          foreground = 'cyan'
+        else
+          foreground = 'black'
+          background = 'red'
+      end
+      # case severity[0..0]
+      #    when 'E'
+      #      color = :redish
+      #    when 'W'
+      #      color = :yellowish
+      #    when 'I'
+      #      color = :purpleish
+      #    when 'D'
+      #      color = :cyanish
+      #    else
+      #      color = :yellowish
+      #  end
+      #  "#{severity[0..0]} [#{datetime.strftime('%H:%M:%S')}#{minion_name}#{description}]  #{msg}\n".send(color)
+      Paint["#{severity[0..0]} [#{datetime.strftime('%H:%M:%S')}#{minion_name}#{description}]  #{msg}\n", foreground, background]
+    end
+    logger
+  end
+  mattr_accessor :logger_package_description
+  mattr_accessor :logger_current_minion
+
+  def self.verbose?
+    return if @filtering_verbose
+    if Pvcglue.command_line_options[:verbose]
+      puts yield
+    end
+  end
+
+  def self.filter_verbose
+    @filtering_verbose = true
+    begin
+      yield
+    ensure
+      @filtering_verbose = false
+    end
+  end
+
+  def self.reset_minion_state?
+    !!Pvcglue.command_line_options[:rebuild]
+  end
+
   def self.gem_dir
     Gem::Specification.find_by_name('pvcglue').gem_dir
   end
@@ -33,8 +118,8 @@ module Pvcglue
   end
 
   def self.render_template(template, file_name = nil)
-    puts '-'*80
-    puts "---> render_template(template=#{template}, file_name=#{file_name}"
+    # puts '-'*80
+    # puts "---> render_template(template=#{template}, file_name=#{file_name}"
     data = Tilt.new(Pvcglue.template_file_name(template)).render
     if file_name
       File.write(file_name, data)
@@ -47,7 +132,7 @@ module Pvcglue
     # puts "Running `#{cmd}`"
 
     unless system cmd
-      raise(Thor::Error, "Error:  #{$?}")
+      raise("Error:  #{$?}")
     end
     true
   end

data/lib/pvcglue/bootstrap.rb

@@ -2,7 +2,7 @@ module Pvcglue
   class Bootstrap
     def self.run(roles)
       # puts "This is where it should bootstrap #{Pvcglue.cloud.stage_name}.  :)"
-      Pvcglue::Packages.apply('bootstrap'.to_sym, :bootstrap, Pvcglue.cloud.nodes_in_stage(roles), 'root')
+      Pvcglue::Packages.apply('bootstrap'.to_sym, :bootstrap, Pvcglue.cloud.minions_filtered(roles), 'root')
     end
   end
 end

data/lib/pvcglue/cli.rb

@@ -6,6 +6,11 @@ module Pvcglue
 
   class CLI < Thor
     class_option :cloud_manager_override
+    class_option :verbose
+    class_option :rebuild
+    class_option :save_before_upload
+    class_option :create_test_cert
+    class_option :force_cert
 
     def initialize(args = [], local_options = {}, config = {})
       super
@@ -48,22 +53,23 @@ module Pvcglue
     method_option :stage, :required => true, :aliases => "-s"
 
     def build(roles = 'all')
-      Pvcglue::Nodes.build(roles)
+      Pvcglue::Stack.build(Pvcglue.cloud.minions, roles)
     end
 
     desc "console", "open rails console"
     method_option :stage, :required => true, :aliases => "-s"
 
     def console(server='web')
-      node = Pvcglue.cloud.find_node(server)
-      node_name = node.keys.first
-      node_data = node.values.first
-      # puts "*"*80
-      # puts node.inspect
-      puts "Connection to #{node_name} (#{node_data[:public_ip]}) as user 'deploy'..."
+      data = Pvcglue.cloud.minions_filtered(server)
+      minion_name = data.keys.first
+      minion = data.values.first
       working_dir = Pvcglue.cloud.deploy_to_app_current_dir
-      system(%(ssh -t deploy@#{node_data[:public_ip]} "cd #{working_dir} && echo 'Starting #{options[:stage].upcase} Rails console in #{working_dir}' && bundle exec rails c #{options[:stage].downcase}"))
+      Pvcglue.logger.warn("Opening Rails console on #{minion_name} (#{minion.public_ip}) as user '#{minion.remote_user_name}'...")
+      Pvcglue.logger.debug("Project root:  #{working_dir}")
 
+      cmd = "cd #{working_dir} && bundle exec rails c #{options[:stage].downcase}"
+      # system(%(ssh -t deploy@#{node_data[:public_ip]} "cd #{working_dir} && echo 'Starting #{options[:stage].upcase} Rails console in #{working_dir}' && bundle exec rails c #{options[:stage].downcase}"))
+      minion.connection.ssh!(minion.remote_user_name, '-t', cmd)
     end
 
     desc "c", "shortcut for console"
@@ -96,8 +102,11 @@ module Pvcglue
 
     def maintenance(mode)
       raise(Thor::Error, "invalid maintenance mode :(  (Hint:  try on or off.)") unless mode.in?(%w(on off))
-      Pvcglue.cloud.maintenance_mode = mode
-      Pvcglue::Packages.apply(:maintenance_mode, :maintenance, Pvcglue.cloud.nodes_in_stage('lb'))
+      # Pvcglue.cloud.maintenance_mode = mode
+      # Pvcglue::Packages.apply(:maintenance_mode, :maintenance, Pvcglue.cloud.nodes_in_stage('lb'))
+      Pvcglue.cloud.minions_filtered('lb').each do |minioin_name, minion|
+        Pvcglue::Packages::MaintenanceMode.apply(minion, {maintenance_mode: mode})
+      end
     end
 
     desc "maint", "enable or disable maintenance mode"
@@ -120,7 +129,7 @@ module Pvcglue
     def bypass(mode)
       raise(Thor::Error, "invalid maintenance bypass mode :(  (Hint:  try on or off.)") unless mode.in?(%w(on off))
       Pvcglue.cloud.bypass_mode = mode
-      Pvcglue::Packages.apply(:bypass_mode, :maintenance, Pvcglue.cloud.nodes_in_stage('lb'))
+      Pvcglue::Packages.apply(:bypass_mode, :maintenance, Pvcglue.cloud.minions_filtered('lb'))
     end
 
     desc "b", "enable or disable maintenance mode bypass for developers"
@@ -135,13 +144,15 @@ module Pvcglue
     method_option :stage, :required => true, :aliases => "-s"
 
     def sh(server='web') # `shell` is a Thor reserved word
-      node = Pvcglue.cloud.find_node(server)
-      node_name = node.keys.first
-      node_data = node.values.first
+      data = Pvcglue.cloud.minions_filtered(server)
+      minion_name = data.keys.first
+      minion = data.values.first
       # puts "*"*80
       # puts node.inspect
-      puts "Connection to #{node_name} (#{node_data[:public_ip]}) as user 'deploy'..."
-      system("ssh -p #{Pvcglue.cloud.port_in_context(:shell)} deploy@#{node_data[:public_ip]}")
+      Pvcglue.logger.warn("Connecting to #{minion_name} (#{minion.public_ip}) as user '#{minion.remote_user_name}'...")
+      # puts "Connection to #{minion_name} (#{minion.public_ip}) as user '#{minion.remote_user_name}'..."
+      # system("ssh -p #{Pvcglue.cloud.port_in_context(:shell)} #{minion.remote_user_name}@#{minion[:public_ip]}")
+      minion.connection.ssh!(minion.remote_user_name, '', '')
     end
 
     desc "s", "shell"
@@ -163,7 +174,7 @@ module Pvcglue
 
     def rake(*tasks)
       if Pvcglue.cloud.stage_name == 'production'
-      # if Pvcglue.cloud.stage_name == 'local'
+        # if Pvcglue.cloud.stage_name == 'local'
         raise(Thor::Error, "\nDidn't think so!\n") unless yes?("\n\nStop!  Think!  Are you sure you want to do this on the #{Pvcglue.cloud.stage_name} stage? (y/N)")
       end
       Pvcglue::Capistrano.rake(tasks)

data/lib/pvcglue/cloud.rb

@@ -1,7 +1,6 @@
 require "active_support"
 require "active_support/core_ext" # for `with_indifferent_access`
 
-
 module Pvcglue
   class Cloud
     attr_accessor :data
@@ -12,6 +11,7 @@ module Pvcglue
     attr_accessor :stage_env
     attr_accessor :passenger_ruby
     attr_accessor :port_in_node_context
+    # attr_accessor :stage_secrets
 
     def data
       ::Pvcglue::Manager.initialize_cloud_data unless @data
@@ -19,7 +19,10 @@ module Pvcglue
     end
 
     def data=(data)
-      @data = data.with_indifferent_access # We may not want this dependency.
+      # @data = data.with_indifferent_access # We may not want this dependency.
+      # @data = data.to_dot # We may not want this dependency.
+      # @data = Hashie::Mash.new(data) # We may not want this dependency.
+      @data = ::SafeMash.new(data)
     end
 
     def current_node
@@ -57,11 +60,11 @@ module Pvcglue
       @stage_name
     end
 
-    def stage
-      # puts data[app_name].inspect
-      # puts data[app_name][:stages].inspect
-      # puts data[app_name][:stages][stage_name].inspect
-      data[app_name][:stages][stage_name]
+    def old_stage
+      # puts project.inspect
+      # puts project[:stages].inspect
+      # puts project[:stages][stage_name].inspect
+      project[:stages][stage_name]
     end
 
     def stage_roles
@@ -83,39 +86,56 @@ module Pvcglue
     end
 
     # find node by full node_name or by matching prefix of node_name
-    def find_node(node_name, raise_error = true)
-      puts "*"*80
-      raise(Thor::Error, "Node not specified.") if node_name.nil? || node_name.empty?
-      return {node_name => nodes_in_stage[node_name]} if nodes_in_stage[node_name]
-      puts "-"*80
-      nodes_in_stage.each do |key, value|
-        puts key
-        return {key => value} if key.start_with?(node_name)
+    def find_minion_by_name(minion_name, raise_error = true)
+      # raise(Thor::Error, "Node not specified.") if node_name.nil? || node_name.empty?
+      raise('Minion not specified.') if minion_name.nil? || minion_name.empty?
+      return {minion_name => minions_filtered[minion_name]} if minions[minion_name]
+      minions.each do |key, value|
+        return {key => value} if key.start_with?(minion_name)
       end
-      raise("Not found:  #{node_name} in #{stage_name}.") if raise_error
+      raise("Not found:  #{minion_name} in #{stage_name}.") if raise_error
       # raise(Thor::Error, "Not found:  #{node_name} in #{stage_name}.")
-    end
-
-    def nodes_in_stage(role_filter = 'all')
-      # puts (stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }).inspect
-      # stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }
-      nodes = stage_roles.each_with_object({}) do |(role, node), nodes|
-        if role_filter == 'all' || role == role_filter
-          nodes.merge!(node)
-        end
+      nil
+    end
+
+    # # find node by full node_name or by matching prefix of node_name
+    # def find_node(node_name, raise_error = true)
+    #   # raise(Thor::Error, "Node not specified.") if node_name.nil? || node_name.empty?
+    #   raise('Node not specified.') if node_name.nil? || node_name.empty?
+    #   return {node_name => nodes_in_stage[node_name]} if nodes_in_stage[node_name]
+    #   nodes_in_stage.each do |key, value|
+    #     return {key => value} if key.start_with?(node_name)
+    #   end
+    #   raise("Not found:  #{node_name} in #{stage_name}.") if raise_error
+    #   # raise(Thor::Error, "Not found:  #{node_name} in #{stage_name}.")
+    # end
+    #
+    def minions_filtered(role_filter = 'all')
+      # # puts (stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }).inspect
+      # # stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }
+      # nodes = stage_roles.each_with_object({}) do |(role, node), nodes|
+      #   if role_filter == 'all' || role == role_filter
+      #     nodes.merge!(node)
+      #   end
+      # end
+      # # puts nodes.inspect
+      # # puts "nodes_in_stage: only first returned"+"!*"*80
+      # # out = {}
+      # # out["memcached"] = nodes["memcached"]
+      # # puts out.inspect
+      # # out
+      if role_filter == 'all'
+        minions
+      else
+        minions.select { |minion_name, minion| minion.has_role?(role_filter) }
       end
-      # puts nodes.inspect
-      # puts "nodes_in_stage: only first returned"+"!*"*80
-      # out = {}
-      # out["memcached"] = nodes["memcached"]
-      # puts out.inspect
-      # out
+
     end
 
     # ENV['PVC_DEPLOY_TO_BASE'] = stage_data[:deploy_to] || '/sites'
     def deploy_to_base_dir
       # stage[:deploy_to] || '/sites' # TODO:  verify if server setup supports `:deploy_to` override
-      Pvcglue.configuration.web_app_base_dir # TODO:  server setup does not yet support `:deploy_to` override, and would have to be refactored at a higher level than stage.
+      web_app_base_dir # TODO:  server setup does not yet support `:deploy_to` override, and would have to be refactored at a higher level than stage.
     end
 
     # ENV['PVC_DEPLOY_TO_APP'] = "#{ENV['PVC_DEPLOY_TO_BASE']}/#{ENV['PVC_APP_NAME']}/#{ENV['PVC_STAGE']}"
@@ -172,28 +192,29 @@ module Pvcglue
     end
 
     def authorized_keys
-      data[app_name][:authorized_keys]
+      project[:authorized_keys]
     end
 
     def ssh_ports
       ports = []
-      from_all = data[app_name][:ssh_allowed_from_all_port].to_i
+      from_all = project[:ssh_allowed_from_all_port].to_i
       ports << from_all if from_all > 0
       ports
     end
 
     def timezone
-      data[app_name][:time_zone] || 'America/Los_Angeles'
+      project[:time_zone] || 'America/Los_Angeles'
     end
 
     def exclude_tables
-      data[app_name][:excluded_db_tables] || ['versions']
+      project[:excluded_db_tables] || ['versions']
     end
 
     def firewall_allow_incoming_on_port
+      raise 'Not used currently > 0.9'
       # These ports allow incoming connections from any ip address
       ports = []
-      from_all = data[app_name][:ssh_allowed_from_all_port].to_i
+      from_all = project[:ssh_allowed_from_all_port].to_i
       ports << from_all if from_all > 0
       ports.concat [80, 443] if current_node.values.first[:allow_public_access]
       ports.concat ["2000:3000"] if stage_name == 'local'
@@ -201,6 +222,7 @@ module Pvcglue
     end
 
     def firewall_allow_incoming_from_ip
+      raise 'Not used currently > 0.9'
       # Incoming connections to any port are allowed from these ip addresses
       addresses = dev_ip_addresses
       addresses.concat(stage_internal_addresses)
@@ -213,11 +235,13 @@ module Pvcglue
     end
 
     def dev_ip_addresses
-      data[app_name][:dev_ip_addresses].values.each_with_object([]) { |address, addresses| addresses << address }
+      return ['127.0.0.1']
+      # TODO:  Add this functionality back in later
+      project[:dev_ip_addresses].values.each_with_object([]) { |address, addresses| addresses << address }
     end
 
     def stage_internal_addresses
-      nodes_in_stage.values.each_with_object([]) do |value, addresses|
+      minions_filtered.values.each_with_object([]) do |value, addresses|
         addresses << value[:public_ip]
         addresses << value[:private_ip] if value[:private_ip]
       end
@@ -244,6 +268,14 @@ module Pvcglue
       stage[:ssl].to_sym || :none
     end
 
+    def ssl_mode_override
+      @ssl_mode_override || ssl_mode
+    end
+
+    def set_ssl_mode_override(mode)
+      @ssl_mode_override = mode
+    end
+
     def lb_nginx_load_balancing_method
       stage[:lb_nginx_load_balancing_method]
     end
@@ -253,51 +285,51 @@ module Pvcglue
     end
 
     def repo_url
-      data[app_name][:repo_url]
+      project[:repo_url]
     end
 
     def dos_conn_limit_per_ip
-      data[app_name][:dos_conn_limit_per_ip] || stage[:dos_conn_limit_per_ip] || "10"
+      project[:dos_conn_limit_per_ip] || stage[:dos_conn_limit_per_ip] || "10"
     end
 
     def dos_rate
-      data[app_name][:dos_rate] || stage[:dos_rate] || "1"
+      project[:dos_rate] || stage[:dos_rate] || "1"
     end
 
     def dos_burst
-      data[app_name][:dos_burst] || stage[:dos_burst] || "30"
+      project[:dos_burst] || stage[:dos_burst] || "30"
     end
 
     def additional_linked_dirs
-      data[app_name][:additional_linked_dirs] || stage[:additional_linked_dirs] || ""
+      project[:additional_linked_dirs] || stage[:additional_linked_dirs] || ""
     end
 
     def client_header_timeout
-      data[app_name][:client_header_timeout] || stage[:client_header_timeout] || "60s"
+      project[:client_header_timeout] || stage[:client_header_timeout] || "60s"
     end
 
     def client_body_timeout
-      data[app_name][:client_body_timeout] || stage[:client_body_timeout] || "60s"
+      project[:client_body_timeout] || stage[:client_body_timeout] || "60s"
     end
 
     def proxy_read_timeout
-      data[app_name][:proxy_read_timeout] || stage[:proxy_read_timeout] || "60s"
+      project[:proxy_read_timeout] || stage[:proxy_read_timeout] || "60s"
     end
 
     def proxy_send_timeout
-      data[app_name][:proxy_send_timeout] || stage[:proxy_send_timeout] || "60s"
+      project[:proxy_send_timeout] || stage[:proxy_send_timeout] || "60s"
     end
 
     def client_max_body_size
-      data[app_name][:client_max_body_size] || stage[:client_max_body_size] || "1m"
+      project[:client_max_body_size] || stage[:client_max_body_size] || "1m"
     end
 
     def swapfile_size
-      data[app_name][:swapfile_size] || stage[:swapfile_size] || 1024
+      project[:swapfile_size] || stage[:swapfile_size] || 1024
     end
 
     def gems
-      data[app_name][:gems] || {}
+      project[:gems] || {}
     end
 
     def db_rebuild
@@ -309,7 +341,7 @@ module Pvcglue
     end
 
     def nginx_config_ssl_path
-      File.join(nginx_config_path, 'ssl')
+      File.join(nginx_config_path, 'ssl', "#{app_and_stage_name}")
     end
 
     def nginx_ssl_crt_file_name
@@ -333,7 +365,7 @@ module Pvcglue
         when :bootstrap, :manager
           port = "22"
         when :env, :build, :shell, :deploy, :maintenance
-          port = data[app_name][:ssh_allowed_from_all_port] || "22"
+          port = project[:ssh_allowed_from_all_port] || "22"
         else
           raise "Context not specified or invalid"
       end
@@ -358,15 +390,170 @@ module Pvcglue
     end
 
     def monit_mailserver
-      data[app_name][:monit_mailserver] || ""
+      project[:monit_mailserver] || ""
     end
 
     def monit_alert
-      data[app_name][:monit_alert] || ""
+      project[:monit_alert] || ""
     end
 
     def monit_disk_usage_threshold
-      stage[:monit_disk_usage_threshold] || data[app_name][:monit_disk_usage_threshold] || "80%"
+      stage[:monit_disk_usage_threshold] || project[:monit_disk_usage_threshold] || "80%"
+    end
+
+    def minion_manager_user_name
+      'manager'
+    end
+
+    def letsencrypt_full
+      '/var/www/letsencrypt_root/.well-known/acme-challenge'
+    end
+
+    def letsencrypt_root
+      '/var/www/letsencrypt_root'
+    end
+
+
+    # ==============================================================================================
+
+    def manager_minion
+      @manager_minion ||= get_manager_minion
+    end
+
+    def get_manager_minion
+      minion = Pvcglue::Minion.new
+      minion.machine_name = 'pvcglue-manager'
+      minion.roles = ['manager']
+      minion.public_ip = Pvcglue.configuration.cloud_manager
+      minion.connection = Pvcglue::Connection.new(minion)
+      # minion.root_users = machine.root_users
+      # minion.users = machine.users
+      # minion.cloud_id = machine.cloud_id
+      minion.remote_user_name = minion_manager_user_name
+      # minion.all_data = data
+      # minion.project = project
+      # minion.stage = stage
+      # minion.cloud_provider = data.cloud_provider
+      # minion.cloud = ::Pvcglue.cloud
+      # minion.cloud_provider.name == 'not-supported'
+      minion
+
+    end
+
+    # def set_manager_as_project
+    #
+    #   # raise('project already initialized') if @project
+    #   # @project = find_or_raise(data.projects, 'pvcglue_manager')
+    #   @app_name_override = 'pvcglue_manager'
+    #   set_stage('manager')
+    # end
+
+    def reload_minions!
+      @data = nil
+      @project = nil
+      @stage = nil
+      @minions = nil
+    end
+
+    def web_app_base_dir
+      # '/sites'
+      "/home/#{minion_user_name}/www"
+    end
+
+
+    def minion_user_name_base
+      project[:user_name_base] || 'deploy'
+    end
+
+    def minion_user_name
+      "#{minion_user_name_base}-#{stage_name}"
+    end
+
+
+    def find_or_raise(data, name, key = 'name')
+      found = data.detect { |item| item[key] == name }
+      # raise(Thor::Error, "Error:  #{name} not found.") unless found
+      unless found
+        puts "name=#{name}, key=#{key}, data="
+        ap data
+        raise("Error:  #{name} not found.")
+      end
+      found
+    end
+
+    def project
+      @project ||= begin
+        get_project
+      end
+    end
+
+    def project_name
+      project.name
+    end
+
+    def get_project
+      find_or_raise(data.projects, @app_name_override || app_name)
+    end
+
+    def stage
+      @stage ||= begin
+        find_or_raise(project.stages, stage_name)
+      end
+    end
+
+    def minions
+      @minions ||= get_minions
+    end
+
+    def find_machine(name)
+      name.tr!('=', '') # "==dev-lb==" ==> "dev-lb"
+      find_or_raise(data.machines, name)
+    end
+
+    def get_minions
+      minions = ::SafeMash.new
+      stage.stack.each do |item|
+        # ap item
+        machine = find_machine(item.machine_name)
+        # ap machine
+        minion = minions[machine.name]
+        # ap minion
+        unless minion
+          # TODO:  check for duplicate roles:  ie. 2 web servers with the same id
+          minion = Pvcglue::Minion.new
+          minion.machine_name = item.machine_name
+          minion.roles = []
+          minion.private_ip = machine.private_ip
+          minion.public_ip = machine.public_ip
+          minion.connection = Pvcglue::Connection.new(minion)
+          minion.root_users = machine.root_users
+          minion.users = machine.users
+          minion.cloud_id = machine.cloud_id
+          minion.remote_user_name = minion_user_name
+          # TODO:  sync all machine options here, automatically
+        end
+        # ap minion
+        # puts "*"*175
+        # ap minion.roles
+        # ap item.role
+        minion.roles << item.role
+        # ap minion.roles
+
+        # if item.role_index?
+        #   minions
+        # end
+        # ap minion
+
+        minion.all_data = data
+        minion.project = project
+        minion.stage = stage
+        minion.cloud_provider = data.cloud_provider
+        minion.cloud = ::Pvcglue.cloud
+
+        minions[machine.name] = minion
+      end
+      # ap minions
+      minions
     end
   end