pvcglue 0.1.5

data/lib/pvcglue/manager.rb

@@ -0,0 +1,137 @@
+require 'pp'
+
+module Pvcglue
+  class Manager < Thor
+
+    desc "bootstrap", "bootstrap"
+
+    def bootstrap
+      Pvcglue::Packages.apply('bootstrap-manager'.to_sym, :manager, self.class.manager_node, 'root', 'manager')
+    end
+
+    desc "push", "push"
+
+    def push
+      Pvcglue::Manager.push_configuration
+    end
+
+    desc "pull", "pull"
+
+    def pull
+      Pvcglue::Packages.apply('manager-pull'.to_sym, :manager, self.class.manager_node, 'pvcglue', 'manager')
+      self.class.clear_cloud_data_cache
+    end
+
+    desc "show", "show manager data"
+
+    def show
+      self.class.initialize_cloud_data
+      pp Pvcglue.cloud.data
+    end
+
+    desc "info", "show manager data"
+
+    def info
+      show
+    end
+
+    desc "s", "run shell"
+
+    def s # `shell` is a Thor reserved word
+      sh
+    end
+
+    desc "shell", "run shell"
+
+    def sh # `shell` is a Thor reserved word
+      working_dir = self.class.manager_dir
+      cloud_manager = Pvcglue.configuration.cloud_manager
+      user_name = self.class.user_name
+      cloud_name = Pvcglue.configuration.cloud_name
+      puts "Connection to #{cloud_name} cloud on manager at (#{cloud_manager}) as user '#{user_name}'..."
+      system(%(ssh #{Pvcglue.cloud.port_in_context(:manager)} -t #{user_name}@#{cloud_manager} "cd #{working_dir} && bash -i"))
+    end
+
+    desc "configure", "configure"
+
+    def configure
+      Pvcglue.configuration.configure_manager
+    end
+
+    # ------------------------------------------------------------------------------------------------------------------
+
+    def self.initialize_cloud_data
+      unless read_cached_cloud_data
+        Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
+        # Pvcglue::Packages.apply('manager-get-config'.to_sym, :manager, manager_node, 'pvcglue') # Can not use package as it causes infinite recursion, we'll just do it manually
+        data = `ssh pvcglue@#{manager_node[:manager][:public_ip]} "cat #{Pvcglue::Manager.manager_file_name}"`
+        # puts "*"*80
+        # puts data
+        # puts "*"*80
+        if data.empty?
+          raise(Thor::Error, "Remote manager file not found (or empty):  #{::Pvcglue::Manager.manager_file_name}")
+        else
+          ::Pvcglue.cloud.data = TOML.parse(data)
+        end
+        write_cloud_data_cache
+      end
+    end
+
+    def self.write_cloud_data_cache
+      File.write(Pvcglue.configuration.cloud_cache_file_name, TOML.dump(Pvcglue.cloud.data))
+    end
+
+    def self.read_cached_cloud_data
+      # TODO:  Expire cache after given interval
+      if File.exists?(Pvcglue.configuration.cloud_cache_file_name)
+        data = File.read(Pvcglue.configuration.cloud_cache_file_name)
+        Pvcglue.cloud.data = TOML.parse(data)
+        return true
+      end
+      false
+    end
+
+    def self.clear_cloud_data_cache
+      Pvcglue.configuration.clear_cloud_cache
+    end
+
+    def self.manager_node
+      {manager: {public_ip: Pvcglue.configuration.cloud_manager}}
+    end
+
+    def self.cloud_data_file_name_base
+      @file_name_base ||= "#{Pvcglue.configuration.cloud_name}.pvcglue.toml"
+    end
+
+    def self.manager_file_name
+      File.join(manager_dir, cloud_data_file_name_base)
+    end
+
+    def self.user_name
+      'pvcglue'
+    end
+
+    def self.home_dir
+      File.join('/home', user_name)
+    end
+
+    def self.authorized_keys_file_name
+      File.join(ssh_dir, 'authorized_keys')
+    end
+
+    def self.ssh_dir
+      File.join(home_dir, '.ssh')
+    end
+
+    def self.manager_dir
+      File.join(home_dir, '.pvc_manager')
+    end
+
+    def self.push_configuration
+      Pvcglue::Packages.apply('manager-push'.to_sym, :manager, manager_node, 'pvcglue', 'manager')
+      clear_cloud_data_cache
+    end
+
+  end
+
+end

data/lib/pvcglue/nodes.rb

@@ -0,0 +1,29 @@
+module Pvcglue
+  class Nodes
+    def self.build(roles_filter)
+      Pvcglue::Nodes.new(roles_filter).run
+    end
+
+    def initialize(roles_filter)
+      @roles_filter = roles_filter
+    end
+
+    def apply_role?(role)
+      @roles_filter == 'all' || role == @roles_filter
+    end
+    
+    def run
+      puts "This is where it should configure the nodes for #{@roles_filter}.  :)"
+
+      %w(lb db web caching).each do |role|
+        if apply_role?(role)
+          Pvcglue::Packages.apply(role.to_sym, :build, Pvcglue.cloud.nodes_in_stage(role))
+        end
+      end
+
+      Pvcglue::Capistrano.capify
+
+    end
+
+  end
+end

data/lib/pvcglue/packages.rb

@@ -0,0 +1,47 @@
+module Pvcglue
+  class Packages
+    def self.apply(package, context, nodes, user = 'deploy', package_filter = nil)
+      # puts nodes.inspect
+      orca_suite = OrcaSuite.init(package_filter)
+      nodes.each do |node, data|
+        orca_node = ::Orca::Node.new(node, data[:public_ip], {user: user, port: Pvcglue.cloud.port_in_context(context)})
+        ::Pvcglue.cloud.current_node = {node => data}
+        begin
+          orca_suite.run(orca_node.name, package.to_s, :apply)
+        ensure
+          ::Pvcglue.cloud.current_node = nil
+          ::Pvcglue.cloud.current_hostname = nil
+        end
+      end
+    end
+  end
+
+  class OrcaSuite
+
+    def self.init(package_filter)
+      ::Orca.verbose(package_filter != 'manager') # show details for all packages except manager, for now
+
+      # Load orca extensions
+      orca_file = File.join(File.dirname(__FILE__), 'all_the_things.rb')
+      ENV['ORCA_FILE'] = orca_file
+      suite = ::Orca::Suite.new
+      suite.load_file(orca_file)
+      packages_loaded = []
+
+      Dir[File.join(Pvcglue::gem_dir, 'lib', 'pvcglue', 'packages', '*.rb')].each do |file|
+        # package filter is used to load the manager package by itself when stage is not specified
+        next if package_filter && package_filter != File.basename(file, ".rb")
+        begin
+          suite.load_file(file)
+        rescue Exception => e
+          puts "Error loading #{file}:  #{e.message}"
+          raise
+        end
+        packages_loaded << File.basename(file, ".rb")
+      end
+      puts "Packages loaded: #{packages_loaded.sort.join(' ')}."
+      suite
+    end
+  end
+
+end

data/lib/pvcglue/packages/bootstrap.rb

@@ -0,0 +1,92 @@
+#=======================================================================================================================
+package 'bootstrap' do
+#=======================================================================================================================
+  depends_on 'time-zone'
+  depends_on 'hostname'
+  depends_on 'htop'
+  depends_on 'ufw'
+  depends_on 'applications_dir'
+  depends_on 'authorized_keys'
+  depends_on 'sshd-config'
+  depends_on 'firewall-config'
+  depends_on 'firewall-enabled'
+  depends_on 'unattended-security-upgrades'
+end
+
+package 'applications_dir' do
+  depends_on 'deploy-user'
+  validate do
+    stat = run("stat --format=%U:%G:%a #{Pvcglue.configuration.web_app_base_dir}").strip
+    stat == 'deploy:deploy:2755'
+  end
+
+  apply do
+    dir = Pvcglue.configuration.web_app_base_dir
+    # used following as a guide for next line: http://capistranorb.com/documentation/getting-started/authentication-and-authorisation/
+    run "mkdir -p #{dir} && chown deploy:deploy #{dir} && umask 0002 && chmod g+s #{dir}"
+  end
+end
+
+package 'deploy-user' do
+  apply do
+    run "mkdir -p ~/.pvc && chmod 600 ~/.pvc"
+    #run 'adduser --disabled-password --gecos "" deploy'
+    run "useradd -d /home/deploy -G sudo -m -U deploy"
+    run "usermod -s /bin/bash deploy"
+    # this next line will also append this every time this is run...which is less than ideal.  But it *should* only get run once per server due to the validate method
+    run "echo 'deploy ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers" # this may be a security issue, and need refactoring, see the end of http://capistranorb.com/documentation/getting-started/authentication-and-authorisation/
+  end
+
+  remove do
+    raise "removing user not supported, yet.  It needs some 'Are you *really* sure?' stuff."
+    # run "userdel -f deploy"
+    # run "rm -rf /home/deploy"
+  end
+
+  validate do
+    sudo('getent passwd deploy') =~ /^deploy:/ &&
+        sudo('groups deploy') =~ /deploy sudo/
+  end
+
+
+end
+
+package 'authorized_keys' do
+
+  file({
+           :template => ::Pvcglue.template_file_name('authorized_keys.erb'),
+           :destination => '/home/deploy/.ssh/authorized_keys',
+           :create_dirs => true,
+           :permissions => 0644,
+           :user => 'deploy',
+           :group => 'deploy'
+       })
+end
+
+package 'sshd-config' do
+
+  file({
+           :template => Pvcglue.template_file_name('sshd_config.erb'),
+           :destination => '/etc/ssh/sshd_config',
+           :create_dirs => false,
+           :permissions => 0644,
+           :user => 'root',
+           :group => 'root'
+       }) { sudo('service ssh restart') }
+
+end
+
+apt_package 'unattended-upgrades'
+
+package 'unattended-security-upgrades' do
+  depends_on 'unattended-upgrades'
+  file({
+           :template => Pvcglue.template_file_name('20auto-upgrades.erb'),
+           :destination => '/etc/apt/apt.conf.d/20auto-upgrades',
+           :create_dirs => false,
+           :permissions => 0644,
+           :user => 'root',
+           :group => 'root'
+       }) { sudo('service unattended-upgrades restart') }
+end
+

data/lib/pvcglue/packages/env.rb

@@ -0,0 +1,80 @@
+package 'env-initialized' do
+  apply do
+    ::Pvcglue::Env.initialize_stage_env
+  end
+end
+
+package 'env-get-stage' do
+  apply do
+    data = run("cat #{::Pvcglue::Env.stage_env_file_name}")
+    ::Pvcglue.cloud.stage_env = TOML.parse(data)
+  end
+
+end
+
+package 'env-set-stage' do
+  apply do
+    data = TOML.dump(Pvcglue.cloud.stage_env)
+    run(%Q[echo '#{data}' | tee #{::Pvcglue::Env.stage_env_file_name} && chmod 600 #{::Pvcglue::Env.stage_env_file_name}])
+  end
+
+end
+
+package 'deploy-to-base' do
+  validate do
+    stat = run("stat --format=%U:%G:%a #{Pvcglue.cloud.deploy_to_app_shared_dir}").strip
+    stat == 'deploy:deploy:2775'
+  end
+
+  apply do
+    # Reference: http://capistranorb.com/documentation/getting-started/authentication-and-authorisation/
+    run "mkdir -p #{Pvcglue.cloud.deploy_to_app_shared_dir}"
+    # sudo "chown deploy:deploy #{ENV['PVC_DEPLOY_TO_BASE']}"
+    run "umask 0002 && chmod g+s #{Pvcglue.cloud.deploy_to_app_shared_dir}"
+  end
+end
+
+package 'app-env' do
+  depends_on 'deploy-to-base'
+  depends_on 'app-env-file'
+end
+
+package 'app-env-file' do
+  depends_on 'env-initialized'
+
+  file({
+           :template => Pvcglue.template_file_name('web.env.erb'),
+           :destination => Pvcglue.cloud.env_file_name,
+           :create_dirs => true,
+           :permissions => 0640 # TODO:  Double check permissions
+       }) do
+    run("touch #{Pvcglue.cloud.restart_txt_file_name}")
+  end
+
+end
+
+
+package 'env-push' do
+  apply do
+    if File.exists?(::Pvcglue.cloud.env_local_file_name)
+      data = File.read(::Pvcglue.cloud.env_local_file_name)
+      run(%Q[echo '#{data}' | tee #{::Pvcglue::Env.stage_env_file_name}])
+      run(%Q[chmod 600 #{::Pvcglue::Env.stage_env_file_name}])
+    else
+      puts "Local env file not found:  #{::Pvcglue.cloud.env_local_file_name}"
+    end
+  end
+end
+
+package 'env-pull' do
+  apply do
+    data = run("cat #{::Pvcglue::Env.stage_env_file_name}")
+    if data.empty?
+      puts "Remote env file not found:  #{::Pvcglue::Env.stage_env_file_name}"
+    else
+      File.write(::Pvcglue.cloud.env_local_file_name, data)
+      puts "Saved as:  #{::Pvcglue.cloud.env_local_file_name}"
+    end
+  end
+end
+

data/lib/pvcglue/packages/firewall.rb

@@ -0,0 +1,48 @@
+# Reference:  http://manpages.ubuntu.com/manpages/precise/en/man8/ufw-framework.8.html
+package 'firewall-config' do
+
+  file({
+           :template => Pvcglue.template_file_name('ufw.rules6.erb'),
+           :destination => '/lib/ufw/user6.rules',
+           :create_dirs => false,
+           :permissions => 0640,
+           :user => 'root',
+           :group => 'root'
+       }) {  }
+
+  file({
+           :template => Pvcglue.template_file_name('ufw.rules.erb'),
+           :destination => '/lib/ufw/user.rules',
+           :create_dirs => false,
+           :permissions => 0640,
+           :user => 'root',
+           :group => 'root'
+       }) { sudo('service ufw restart') }
+
+end
+
+package 'firewall-enabled' do
+  validate do
+    result = sudo('ufw status verbose')
+    result =~ /Status: active/ && result =~ /Default: deny \(incoming\), allow \(outgoing\)/
+  end
+
+  apply do
+    sudo('ufw --force enable')
+  end
+end
+
+# TODO:  add command line command for this
+package 'update-firewall' do
+  # quick update of firewall settings only.  Full bootstrap must be performed first.
+  depends_on 'firewall-config'
+  depends_on 'firewall-enabled'
+end
+
+
+# TODO:  add command line command for this
+package 'firewall-status' do
+  apply do
+    run "ufw status verbose"
+  end
+end

data/lib/pvcglue/packages/manager.rb

@@ -0,0 +1,102 @@
+require 'toml'
+apt_package 'htop'
+apt_package 'ufw'
+
+package 'bootstrap-manager' do
+  # TODO: firewall and ssh port config
+  #depends_on 'time-zone'
+  depends_on 'htop'
+  # depends_on 'ufw'
+  #depends_on 'deploy-user'
+  #depends_on 'sshd-config'
+  #depends_on 'firewall-config'
+  depends_on 'pvcglue-user'
+  depends_on 'manager-copy-id'
+end
+
+package 'pvcglue-user' do
+  apply do
+    # Local variables used to improve readability of bash commands :)
+    user_name = Pvcglue::Manager.user_name
+    home_dir = Pvcglue::Manager.home_dir
+    manager_dir = Pvcglue::Manager.manager_dir
+    ssh_dir = Pvcglue::Manager.ssh_dir
+
+    sudo "useradd -d #{home_dir} -m -U #{user_name}"
+    sudo "usermod -s /bin/bash #{user_name}"
+    sudo "mkdir -p #{manager_dir} && chown #{user_name}:#{user_name} #{manager_dir} && chmod 700 #{manager_dir}"
+    sudo "mkdir -p #{ssh_dir} && chown #{user_name}:#{user_name} #{ssh_dir} && chmod 700 #{ssh_dir}"
+  end
+
+  remove do
+    raise "removing user not supported, yet.  It needs some 'Are you *really* sure?' stuff."
+    # user_name = Pvcglue::Manager.user_name
+    # home_dir = Pvcglue::Manager.home_dir
+    #sudo "userdel -f #{user_name}"
+    #sudo "rm -rf #{home_dir}"
+  end
+
+  validate do
+    user_name = Pvcglue::Manager.user_name
+    # home_dir = Pvcglue::Manager.home_dir
+    #sudo "userdel -f #{user_name}"; sudo "rm -rf #{home_dir}"; raise "User has been deleted"
+    sudo("getent passwd #{user_name}") =~ /^#{user_name}:/
+  end
+end
+
+package 'manager-copy-id' do
+  validate do
+    authorized_keys_file_name = Pvcglue::Manager.authorized_keys_file_name
+    user_key = `cat ~/.ssh/id_rsa.pub`.strip
+    auth = run("cat #{authorized_keys_file_name}")
+    auth.include?(user_key)
+  end
+
+  apply do
+    authorized_keys_file_name = Pvcglue::Manager.authorized_keys_file_name
+    user_name = Pvcglue::Manager.user_name
+    copy_id = %Q[cat ~/.ssh/id_rsa.pub | ssh #{node.get(:user)}@#{node.host} "cat >> #{authorized_keys_file_name}"]
+    system "#{copy_id}"
+    run(%Q[cat "" >> #{authorized_keys_file_name}])
+    sudo "chown #{user_name}:#{user_name} #{authorized_keys_file_name} && chmod 600 #{authorized_keys_file_name}"
+  end
+end
+
+package 'manager-push' do
+  apply do
+    if File.exists?(::Pvcglue.cloud.local_file_name)
+      data = File.read(::Pvcglue.cloud.local_file_name)
+      run(%Q[echo '#{data}' | tee #{::Pvcglue::Manager.manager_file_name}])
+      run(%Q[chmod 600 #{::Pvcglue::Manager.manager_file_name}])
+    else
+      puts "Local file not found:  #{::Pvcglue.cloud.local_file_name}"
+    end
+  end
+end
+
+package 'manager-pull' do
+  apply do
+    data = run("cat #{::Pvcglue::Manager.manager_file_name}")
+    if data.empty?
+      puts "Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}"
+    else
+      File.write(::Pvcglue.cloud.local_file_name, data)
+      puts "Saved as:  #{::Pvcglue.cloud.local_file_name}"
+    end
+  end
+end
+
+package 'manager-get-config' do
+  apply do
+    data = run("cat #{::Pvcglue::Manager.manager_file_name}")
+    #puts "*"*80
+    #puts data
+    #puts "*"*80
+    if data.empty?
+      raise "Remote manager file not found:  #{::Pvcglue::Manager.manager_file_name}"
+    else
+      ::Pvcglue.cloud.data = TOML.parse(data)
+    end
+  end
+end
+