pvcglue 0.1.39 → 0.9.0

data/lib/pvcglue/manager.rb

@@ -7,7 +7,9 @@ module Pvcglue
     desc "bootstrap", "bootstrap"
 
     def bootstrap
-      Pvcglue::Packages.apply('bootstrap-manager'.to_sym, :manager, self.class.manager_node, 'root', 'manager')
+      # Pvcglue.cloud.set_manager_as_project
+      Pvcglue::Stack.build({'pvcglue-manager' => Pvcglue.cloud.manager_minion}, 'manager')
+      # Pvcglue::Packages.apply('bootstrap-manager'.to_sym, :manager, self.class.manager_node, 'root', 'manager')
     end
 
     desc "push", "push"
@@ -19,15 +21,16 @@ module Pvcglue
     desc "pull", "pull"
 
     def pull
-      Pvcglue::Packages.apply('manager-pull'.to_sym, :manager, self.class.manager_node, 'pvcglue', 'manager')
-      self.class.clear_cloud_data_cache
+      Pvcglue::Manager.pull_configuration
+      # Pvcglue::Packages.apply('manager-pull'.to_sym, :manager, self.class.manager_node, 'pvcglue', 'manager')
+      # self.class.clear_cloud_data_cache
     end
 
     desc "show", "show manager data"
 
     def show
       self.class.initialize_cloud_data
-      pp Pvcglue.cloud.data
+      ap Pvcglue.cloud.data
     end
 
     desc "info", "show manager data"
@@ -100,21 +103,23 @@ module Pvcglue
     end
 
     def self.initialize_cloud_data
-      # return if get_local_cloud_data
-      unless read_cached_cloud_data
-        Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
-        # Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue') # Can not use package as it causes infinite recursion, we'll just do it manually
-        data = `ssh pvcglue@#{manager_node[:manager][:public_ip]} "cat #{Pvcglue::Manager.manager_file_name}"`
-        # puts "*"*80
-        # puts data
-        # puts "*"*80
-        if data.empty?
-          raise(Thor::Error, "Remote manager file not found (or empty):  #{::Pvcglue::Manager.manager_file_name}")
-        else
-          ::Pvcglue.cloud.data = TOML.parse(data)
-        end
-        write_cloud_data_cache
-      end
+      Pvcglue::Packages::Manager.get_configuration
+
+      # # return if get_local_cloud_data
+      # unless read_cached_cloud_data
+      #   Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
+      #   # Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue') # Can not use package as it causes infinite recursion, we'll just do it manually
+      #   data = `ssh pvcglue@#{manager_node[:manager][:public_ip]} "cat #{Pvcglue::Manager.manager_file_name}"`
+      #   # puts "*"*80
+      #   # puts data
+      #   # puts "*"*80
+      #   if data.empty?
+      #     raise(Thor::Error, "Remote manager file not found (or empty):  #{::Pvcglue::Manager.manager_file_name}")
+      #   else
+      #     ::Pvcglue.cloud.data = TOML.parse(data)
+      #   end
+      #   write_cloud_data_cache
+      # end
     end
 
     # def self.get_local_cloud_data
@@ -127,7 +132,7 @@ module Pvcglue
     # end
 
     def self.write_cloud_data_cache
-      File.write(Pvcglue.configuration.cloud_cache_file_name, TOML.dump(Pvcglue.cloud.data))
+      # File.write(Pvcglue.configuration.cloud_cache_file_name, TOML.dump(Pvcglue.cloud.data))
     end
 
     def self.read_cached_cloud_data
@@ -163,11 +168,13 @@ module Pvcglue
     end
 
     def self.user_name
-      'pvcglue'
+      raise('Old username should not be used')
+      # 'pvcglue'
     end
 
     def self.home_dir
-      File.join('/home', user_name)
+      # File.join('/home', user_name)
+      '~'
     end
 
     def self.authorized_keys_file_name
@@ -179,12 +186,18 @@ module Pvcglue
     end
 
     def self.manager_dir
-      File.join(home_dir, '.pvc_manager')
+      File.join(home_dir, 'manager')
     end
 
     def self.push_configuration
-      Pvcglue::Packages.apply('manager-push'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
-      clear_cloud_data_cache
+      Pvcglue::Packages::Manager.push_configuration
+      # Pvcglue::Packages.apply('manager-push'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
+      # clear_cloud_data_cache
+    end
+
+    def self.pull_configuration
+      Pvcglue::Packages::Manager.pull_configuration
+      # clear_cloud_data_cache
     end
 
     def self.local_mode?

data/lib/pvcglue/minion.rb

@@ -0,0 +1,182 @@
+module Pvcglue
+  class Minion < SafeMash
+    include Hashie::Extensions::Mash::SafeAssignment
+
+    def build!
+      # puts '*'*175
+      Pvcglue.logger.info('BUILD') { "Building #{machine_name}" }
+
+      minion = self # for readability
+      Pvcglue::Packages::SshKeyCheck.apply(minion)
+      Pvcglue::Packages::AptRepos.apply(minion)
+      Pvcglue::Packages::AptUpdate.apply(minion)
+      Pvcglue::Packages::AptUpgrade.apply(minion)
+      Pvcglue::Packages::Swap.apply(minion)
+      Pvcglue::Packages::Apt.apply(minion)
+      Pvcglue::Packages::Firewall.apply(minion)
+      Pvcglue::Packages::UnattendedUpgrades.apply(minion)
+      Pvcglue::Packages::Users.apply(minion)
+      Pvcglue::Packages::AuthorizedKeys.apply(minion)
+      Pvcglue::Packages::Roles.apply(minion)
+
+
+      # puts '='*175
+    end
+
+    # def build_manager!
+    #   Pvcglue.logger.info('MANAGER') { "Building #{machine_name}" }
+    #   Pvcglue.cloud.set_manager_as_project
+    #   build!
+    # end
+
+    def provision!
+      create! unless provisioned?
+    end
+
+    def provisioned?
+      public_ip.present?
+    end
+
+    def create!
+      raise(Thor::Error, "#{cloud_provider.name} unknown") unless cloud_provider.name == 'digital-ocean'
+      Pvcglue.logger.warn("Provisioning a machine for #{machine_name} on #{cloud_provider.name}...")
+
+      # TODO:  Tags.  production, staging, load-balancer, web, worker, database, postgress, cache, memcache...
+      name = machine_name
+      size = capacity || cloud_provider.default_capacity
+      image = cloud_provider.image
+      region = cloud_provider.region
+      # ap cloud_provider.initial_users
+      # ap cloud_provider
+      ssh_keys = cloud_provider.initial_users.map { |description, ssh_key| ssh_key }
+      backups = cloud_provider.backups.nil? ? true : cloud_provider.backups # default to true -- safety first!
+      tags = cloud_provider.tags
+
+      # client.droplets.all.each do |droplet|
+      #   ap droplet
+      # end
+      droplet = DropletKit::Droplet.new(
+          name: name,
+          region: region,
+          image: image,
+          size: size,
+          ssh_keys: ssh_keys,
+          backups: backups,
+          ipv6: false,
+          private_networking: true,
+          user_data: '',
+          monitoring: true,
+          tags: tags
+      )
+      droplet = Pvcglue::DigitalOcean.client.droplets.create(droplet)
+      self.droplet = droplet
+      Pvcglue.logger.debug("Droplet ID:  #{droplet.id}")
+      true
+    end
+
+    def has_role?(roles)
+      return true if roles == 'all'
+      !(Array(roles).map(&:to_sym) & roles_to_sym).empty?
+    end
+
+    def has_roles?(roles)
+      has_role?(roles)
+    end
+
+    def roles_to_sym
+      roles.map(&:to_sym)
+    end
+
+    def minion_manager?
+      has_role?('manager')
+    end
+
+    def get_user(user_name)
+      Pvcglue.cloud.data.users.detect { |user| user.name == user_name }
+    end
+
+    def get_users_from_group(names)
+      names = Array(names)
+      users = []
+      names.each do |name|
+        if name =~ /\A==.*==\z/
+          group = Pvcglue.cloud.data.groups[name[2..-3]]
+          # ap group
+          users.concat(get_users_from_group(group))
+        else
+          users << get_user(name)
+        end
+      end
+      users
+    end
+
+    def get_root_users
+      if minion_manager?
+        get_users_from_group('==manager_root_users==')
+      else
+        get_users_from_group('==lead_developers==')
+      end
+    end
+
+    def get_users
+      if minion_manager?
+        get_users_from_group('==manager_users==')
+      else
+        get_users_from_group('==developers==')
+      end
+    end
+
+    def get_root_authorized_keys_data
+      get_authorized_keys_data(get_root_users)
+    end
+
+    def get_users_authorized_keys_data
+      get_authorized_keys_data(get_users)
+    end
+
+    # def get_root_authorized_keys
+    #   get_authorized_keys(get_root_users)
+    # end
+    #
+    # def get_users_authorized_keys
+    #   get_authorized_keys(get_users)
+    # end
+    #
+    # def get_authorized_keys(users)
+    #   keys = []
+    #   users.each do |user|
+    #     user.public_keys.each do |id, public_key|
+    #       keys << public_key
+    #     end
+    #   end
+    #   keys
+    # end
+
+    def get_github_authorized_keys(user)
+      return [] unless user.github_user_name.present?
+      uri = URI("https://github.com/#{user.github_user_name}.keys")
+      Net::HTTP.get(uri).split("\n")
+    end
+
+    def get_authorized_keys_data(users)
+      data = []
+      users.each do |user|
+        keys = []
+        keys += user.public_keys.values if user.public_keys
+        keys += get_github_authorized_keys(user)
+        keys.each do |public_key|
+          line = %Q(environment="PVCGLUE_USER=#{user.name}" #{public_key})
+          # line = %Q(command="export PVCGLUE_USER=#{id}" #{public_key})
+          data << line
+        end
+      end
+      data
+    end
+
+    # def update_from(another_minion)
+    #   self.private_ip = another_minion.private_ip
+    #   self.public_ip = another_minion.public_ip
+    #   self.cloud_id = another_minion.cloud_id
+    # end
+  end
+end

data/lib/pvcglue/nodes.rb

@@ -17,7 +17,7 @@ module Pvcglue
 
       %w(lb db web caching redis).each do |role|
         if apply_role?(role)
-          Pvcglue::Packages.apply(role.to_sym, :build, Pvcglue.cloud.nodes_in_stage(role))
+          Pvcglue::Packages.apply(role.to_sym, :build, Pvcglue.cloud.minions_filtered(role))
         end
       end
 

data/lib/pvcglue/{packages → old_packages}/bootstrap.rb

@@ -1,20 +1,20 @@
 #=======================================================================================================================
 package 'bootstrap' do
 #=======================================================================================================================
-  depends_on 'time-zone'
-  depends_on 'hostname'
-  depends_on 'htop'
-  depends_on 'ufw'
-  depends_on 'applications_dir'
-  depends_on 'authorized_keys_root'
-  depends_on 'authorized_keys'
-  depends_on 'sshd-config'
-  depends_on 'firewall-config'
-  depends_on 'firewall-enabled'
-  depends_on 'unattended-security-upgrades'
+  depends_on 'time-zone' # Later, if needed
+  depends_on 'hostname' # Later, if needed
+  depends_on 'htop' # DONE
+  depends_on 'ufw' # DONE
+  depends_on 'applications_dir' # DONE
+  depends_on 'authorized_keys_root' # DONE
+  depends_on 'authorized_keys' # DONE
+  depends_on 'sshd-config' # Later, if needed
+  depends_on 'firewall-config' # DONE
+  depends_on 'firewall-enabled' # DONE
+  depends_on 'unattended-security-upgrades' # DONE
 end
 
-package 'applications_dir' do
+package 'applications_dir' do # DONE
   depends_on 'deploy-user'
   validate do
     stat = run("stat --format=%U:%G:%a #{Pvcglue.configuration.web_app_base_dir}").strip
@@ -28,7 +28,7 @@ package 'applications_dir' do
   end
 end
 
-package 'deploy-user' do
+package 'deploy-user' do # DONE
   apply do
     run "mkdir -p ~/.pvc && chmod 600 ~/.pvc"
     #run 'adduser --disabled-password --gecos "" deploy'
@@ -52,7 +52,7 @@ package 'deploy-user' do
 
 end
 
-package 'authorized_keys' do
+package 'authorized_keys' do # DONE
 
   file({
            :template => ::Pvcglue.template_file_name('authorized_keys.erb'),
@@ -64,7 +64,7 @@ package 'authorized_keys' do
        })
 end
 
-package 'authorized_keys_root' do
+package 'authorized_keys_root' do # DONE
 
   file({
            :template => ::Pvcglue.template_file_name('authorized_keys.erb'),

data/lib/pvcglue/{packages → old_packages}/env.rb

@@ -1,10 +1,10 @@
-package 'env-initialized' do
+package 'env-initialized' do # DONE
   apply do
     ::Pvcglue::Env.initialize_stage_env
   end
 end
 
-package 'env-get-stage' do
+package 'env-get-stage' do # DONE
   apply do
     data = run("cat #{::Pvcglue::Env.stage_env_file_name}")
     ::Pvcglue.cloud.stage_env = TOML.parse(data)
@@ -12,7 +12,7 @@ package 'env-get-stage' do
 
 end
 
-package 'env-set-stage' do
+package 'env-set-stage' do # DONE
   apply do
     data = TOML.dump(Pvcglue.cloud.stage_env)
     run(%Q[echo '#{data}' | tee #{::Pvcglue::Env.stage_env_file_name} && chmod 600 #{::Pvcglue::Env.stage_env_file_name}])
@@ -20,7 +20,7 @@ package 'env-set-stage' do
 
 end
 
-package 'deploy-to-base' do
+package 'deploy-to-base' do # DONE
   validate do
     stat = run("stat --format=%U:%G:%a #{Pvcglue.cloud.deploy_to_app_shared_dir}").strip
     stat == 'deploy:deploy:2775'
@@ -35,11 +35,11 @@ package 'deploy-to-base' do
 end
 
 package 'app-env' do
-  depends_on 'deploy-to-base'
+  depends_on 'deploy-to-base' # DONE
   depends_on 'app-env-file'
 end
 
-package 'app-env-file' do
+package 'app-env-file' do # DONE
   depends_on 'env-initialized'
 
   file({
@@ -54,7 +54,7 @@ package 'app-env-file' do
 end
 
 
-package 'env-push' do
+package 'env-push' do # LATER
   apply do
     if File.exists?(::Pvcglue.cloud.env_local_file_name)
       data = File.read(::Pvcglue.cloud.env_local_file_name)
@@ -66,7 +66,7 @@ package 'env-push' do
   end
 end
 
-package 'env-pull' do
+package 'env-pull' do # LATER
   apply do
     data = run("cat #{::Pvcglue::Env.stage_env_file_name}")
     if data.empty?

data/lib/pvcglue/old_packages/firewall.rb

@@ -0,0 +1,48 @@
+# Reference:  http://manpages.ubuntu.com/manpages/precise/en/man8/ufw-framework.8.html
+package 'firewall-config' do
+
+  file({
+           :template => Pvcglue.template_file_name('ufw.rules6.erb'),
+           :destination => '/lib/ufw/user6.rules',
+           :create_dirs => false,
+           :permissions => 0640,
+           :user => 'root',
+           :group => 'root'
+       }) {  }
+
+  file({
+           :template => Pvcglue.template_file_name('ufw.rules.erb'),
+           :destination => '/lib/ufw/user.rules',
+           :create_dirs => false,
+           :permissions => 0640,
+           :user => 'root',
+           :group => 'root'
+       }) { sudo('service ufw restart') }
+
+end
+
+package 'firewall-enabled' do
+  validate do
+    result = sudo('ufw status verbose')
+    result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
+  end
+
+  apply do
+    sudo('ufw --force enable')
+  end
+end
+
+# TODO:  add command line command for this
+package 'update-firewall' do
+  # quick update of firewall settings only.  Full bootstrap must be performed first.
+  depends_on 'firewall-config'
+  depends_on 'firewall-enabled'
+end
+
+
+# TODO:  add command line command for this
+package 'firewall-status' do
+  apply do
+    run "ufw status verbose"
+  end
+end