subspace 2.5.10 → 3.0.0

data/lib/subspace/commands/ansible.rb

@@ -1,19 +1,28 @@
 module Subspace
   module Commands
     module Ansible
+      def ansible_playbook(*args)
+        args.push "--diff"
+        ansible_command("ansible-playbook", *args)
+      end
+
       def ansible_command(command, *args)
         update_ansible_cfg
-        Dir.chdir "config/provision" do
+        retval = false
+        Dir.chdir "config/subspace" do
           say ">> Running #{command} #{args.join(' ')}"
-          system(command, *args, out: $stdout, err: $stderr)
+          retval = system(command, *args, out: $stdout, err: $stderr)
           say "<< Done"
         end
+        retval
       end
 
       private
 
       def update_ansible_cfg
-        if !ENV["DISABLE_MITOGEN"] && `pip show mitogen 2>&1` =~ /^Location: (.*?)$/m
+        if ENV["DISABLE_MITOGEN"]
+          puts "Mitogen explicitly disabled.  Skipping detection. "
+        elsif `pip show mitogen 2>&1` =~ /^Location: (.*?)$/m
           @mitogen_path = $1
           puts "🏎🚀🚅Mitogen found at #{@mitogen_path}.  WARP 9!....ENGAGE!🚀"
         else

data/lib/subspace/commands/base.rb

@@ -13,7 +13,7 @@ module Subspace
       end
 
       def template_dir
-        File.join(gem_path, 'template', 'provision')
+        File.join(gem_path, 'template', 'subspace')
       end
 
       def gem_path
@@ -21,23 +21,34 @@ module Subspace
       end
 
       def project_path
+        unless File.exist?(File.join(Dir.pwd, "config", "subspace"))
+          say "Subspace must be run from the project root"
+          exit
+        end
         Dir.pwd # TODO make sure this is correct if they for whatever reason aren't running subspace from the project root??
       end
 
+      def project_name
+        File.basename(project_path) # TODO see above, this should probably be in a configuration somewhere
+      end
+
       def dest_dir
-        "config/provision"
+        "config/subspace"
       end
 
       def template(src, dest = nil, render_binding = nil)
         return unless confirm_overwrite File.join(dest_dir, dest || src)
         template! src, dest, render_binding
-        say "Wrote #{dest}"
+        say "Wrote #{dest || src}"
       end
 
       def template!(src, dest = nil, render_binding = nil)
         dest ||= src
-        template = ERB.new File.read(File.join(template_dir, "#{src}.erb")), nil, '-'
-        File.write File.join(dest_dir, dest), template.result(render_binding || binding)
+        template = ERB.new File.read(File.join(template_dir, "#{src}.erb")), trim_mode: '-'
+        result = template.result(render_binding || binding)
+
+
+        File.write File.join(dest_dir, dest), result
       end
 
       def copy(src, dest = nil)
@@ -74,6 +85,10 @@ module Subspace
       def set_subspace_version
         ENV['SUBSPACE_VERSION'] = Subspace::VERSION
       end
+
+      def inventory
+        @inventory ||= Subspace::Inventory.read("config/subspace/inventory.yml")
+      end
     end
   end
 end

data/lib/subspace/commands/bootstrap.rb

@@ -1,5 +1,4 @@
 class Subspace::Commands::Bootstrap < Subspace::Commands::Base
-  PASS_THROUGH_PARAMS = ["private-key"]
 
   def initialize(args, options)
     @host_spec = args.first
@@ -11,37 +10,33 @@ class Subspace::Commands::Bootstrap < Subspace::Commands::Base
 
   def run
     # ansible atlanta -m copy -a "src=/etc/hosts dest=/tmp/hosts"
-    install_python
-    ensure_ssh_dir
+    hosts = inventory.find_hosts!(@host_spec)
+    update_ansible_cfg
+    hosts.each do |host|
+      say "Bootstapping #{host.vars["hostname"]}..."
+      learn_host(host)
+      install_python(host)
+    end
   end
 
   private
 
-  def ensure_ssh_dir
-    cmd = ["ansible",
-      @host_spec,
-      "-m",
-      "file",
-      "-a",
-      "path=/home/{{ansible_ssh_user}}/.ssh state=directory mode=0700",
-      "-vvvv"
-    ]
-    cmd = cmd | pass_through_params
-    bootstrap_command cmd
+  def learn_host(host)
+    system "ssh-keygen -R #{host.vars["ansible_host"]}"
+    system "ssh-keyscan -H #{host.vars["ansible_host"]} >> ~/.ssh/known_hosts"
   end
 
-  def install_python
-    update_ansible_cfg
+  def install_python(host)
     cmd = ["ansible",
-      @host_spec,
+      host.name,
+      "--private-key",
+      "config/subspace/subspace.pem",
       "-m",
       "raw",
       "-a",
-      "test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)",
-      "--become",
-      "-vvvv"
+      "test -e /usr/bin/python3 || (apt -y update && apt install -y python3)",
+      "--become"
     ]
-    cmd = cmd | pass_through_params
     bootstrap_command cmd
   end
 

data/lib/subspace/commands/configure.rb

@@ -16,12 +16,12 @@ class Subspace::Commands::Configure < Subspace::Commands::Base
   private
 
   def update_host_configuration(host)
-    say "Generating config/provisiong/host_vars/#{host}"
+    say "Generating config/subspace/host_vars/#{host}"
     template "host_vars/template", "host_vars/#{host}", Subspace.config.binding_for(host: host)
   end
 
   def update_group_configuration(group)
-    say "Generating config/provisiong/group_vars/#{group}"
+    say "Generating config/subspace/group_vars/#{group}"
     template "group_vars/template", "group_vars/#{group}", Subspace.config.binding_for(group: group)
   end
 end

data/lib/subspace/commands/exec.rb

@@ -0,0 +1,20 @@
+require 'yaml'
+require 'subspace/inventory'
+class Subspace::Commands::Exec < Subspace::Commands::Base
+  PASS_THROUGH_PARAMS = ["i"]
+
+  def initialize(args, options)
+    @host_spec = args[0]
+    @command = args[1]
+    @user = options.user
+    @options = options
+    run
+  end
+
+  def run
+    hosts = inventory.find_hosts!(@host_spec)
+
+    say "> Running `#{@command}` on #{hosts.join ','}"
+    ansible_command "ansible", @host_spec, "-m", "command", "-a", @command
+  end
+end

data/lib/subspace/commands/init.rb

@@ -3,76 +3,113 @@ require 'erb'
 require 'securerandom'
 class Subspace::Commands::Init < Subspace::Commands::Base
   def initialize(args, options)
-    if args.first == "vars"
-      init_vars
-    else
-      run
+    options.default env: "dev"
+
+    @env = options.env
+    @template = options.template
+
+    if options.ansibe.nil? && options.terraform.nil?
+      # They didn't pass in any options (subspace init) so just do both
+      options.ansible = true
+      options.terraform = true
+    end
+
+    if @template.nil? && options.terraform == true
+      answer = ask "What template/server configuration would you like to use? e.g. 'workhorse' or 'oxenwagen'"
+      @template = answer
     end
+
+    @init_ansible = options.ansible
+    @init_terraform = options.terraform
+    run
   end
 
   def run
     if File.exists? dest_dir
       answer = ask "Subspace appears to be initialized.  Reply 'yes' to continue anyway: [no] "
       abort unless answer.chomp == "yes"
+    else
+      FileUtils.mkdir_p dest_dir
     end
 
-    FileUtils.mkdir_p File.join dest_dir, "group_vars"
-    FileUtils.mkdir_p File.join dest_dir, "host_vars"
-    FileUtils.mkdir_p File.join dest_dir, "vars"
-    FileUtils.mkdir_p File.join dest_dir, "roles"
+    init_pemfile
+    init_ansible if @init_ansible
+    init_terraform if @init_terraform
 
-    copy ".gitignore"
-    #template "../provision.rb"
-    template "ansible.cfg"
-    template "hosts"
-    template "group_vars/all"
+    puts """
+    1. Inspect key config files:
+      - config/subspace/terraform/#{@env}/main.tf     # Main terraform file
+      - config/subspace/#{@env}.yml                   # Main ansible playbook
+      - config/subspace/group_vars                    # Ansible configuration options
+      - config/subspace/inventory.yml                 # Server Inventory
+      - config/subspace/templates/authorized_keys     # SSH Authorized Keys
+      - config/subspace/templates/application.yml     # Application Environment variables
 
-    create_vault_pass
-    environments.each do |env|
-      @env = env
-      @hostname = hostname(env)
-      template "group_vars/template", "group_vars/#{env}"
-      template "host_vars/template", "host_vars/#{env}"
-      create_vars_file_for_env env
-      template "playbook.yml", "#{env}.yml"
-    end
-    create_vars_file_for_env "development"
-    init_vars
+    2. create cloud infrastructure with terraform:
 
-    puts """
-    1. Create a server.
+      subspace tf #{@env}
 
-    2. Set your server's location:
+    3. Bootstrap the new server
 
-      vim config/provision/host_vars/production
-      vim config/provision/host_vars/dev
+      subspace boostrap #{@env}1
 
-    3. Set up your authorized_keys:
+    4. Inspect new environment
+      - ensure the correct roles are present in #{@env}.yml
+      - Check ansible configuration variables in group_vars/#{@env}
 
-      vim config/provision/authorized_keys
+    4. Provision the new servers with ansible:
 
-    4. Then provision your server:
+      subspace provision #{@env}
 
-      subspace provision dev
-      subspace provision production
+    !!MAKE SURE YOU PUT config/subspace/subspace.pem SOMEWHERE!!
 
   """
 
   end
 
-  def init_vars
-    FileUtils.mkdir_p File.join dest_dir, "templates"
-    copy "templates/application.yml.template"
+  private
+
+  def init_pemfile
+    pem = File.join dest_dir, "subspace.pem"
+    if File.exist?(pem)
+      say "Existing SSH Keypair exists.  Skipping keygen."
+    else
+      say "Creating SSH Keypair in #{pem}"
+      `ssh-keygen -t rsa -f #{pem}`
+    end
   end
 
-  private
+  def init_ansible
+    FileUtils.mkdir_p File.join dest_dir, "group_vars"
+    FileUtils.mkdir_p File.join dest_dir, "secrets"
+    FileUtils.mkdir_p File.join dest_dir, "roles"
+    FileUtils.mkdir_p File.join dest_dir, "templates"
+
+    copy ".gitignore"
+    template "ansible.cfg"
+    template "group_vars/all"
+    template "inventory.yml"
+    template "templates/authorized_keys"
 
-  def project_name
-    File.basename(Dir.pwd)
+    create_vault_pass
+    @hostname = hostname(@env)
+    create_secrets_for @env
+    template "group_vars/template", "group_vars/#{@env}"
+    template "playbook.yml", "#{@env}.yml"
+
+    create_secrets_for "development" #TODO rename to local?
+    init_appyml
   end
 
-  def environments
-    %w(production dev)
+  def init_terraform
+    Subspace::Commands::Terraform.ensure_terraform
+    Subspace::Commands::Terraform.check_aws_credentials(project_name)
+
+    FileUtils.mkdir_p File.join dest_dir, "terraform", @env
+
+    set_latest_ami
+    template "terraform/template/main-#{@template}.tf", "terraform/#{@env}/main.tf"
+    copy "terraform/.gitignore"
   end
 
   def hostname(env)
@@ -87,10 +124,22 @@ class Subspace::Commands::Init < Subspace::Commands::Base
     end
   end
 
-  def create_vars_file_for_env(env)
-    template "vars/template", "vars/#{env}.yml"
+  def create_secrets_for(env)
+    template "secrets/template", "secrets/#{env}.yml"
     Dir.chdir dest_dir do
-      `ansible-vault encrypt vars/#{env}.yml`
+      `ansible-vault encrypt secrets/#{env}.yml`
     end
   end
+
+  def init_appyml
+    copy "templates/application.yml.template"
+  end
+
+  def set_latest_ami
+    @latest_ami = `aws --profile subspace-#{project_name} ec2 describe-images \
+    --filters 'Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64*' \
+    --query 'Images[*].[ImageId,CreationDate]' --output text \
+    | sort -k2 -r \
+    | head -n1 | cut -f1`.chomp
+  end
 end

data/lib/subspace/commands/inventory.rb

@@ -0,0 +1,54 @@
+class Subspace::Commands::Inventory < Subspace::Commands::Base
+
+  def initialize(args, options)
+    command = args.first
+    @env = options.env
+    case command
+    when "capistrano"
+      capistrano_deployrb
+    when "list"
+      list_inventory
+    when "keyscan"
+      keyscan_inventory
+    else
+      say "Unknown or missing command to inventory: #{command}"
+      say "try subspace inventory [list, capistrano]"
+    end
+  end
+
+  def list_inventory
+    inventory.find_hosts!(@env || "all").each do |host|
+      puts "#{host.name}\t#{host.vars["ansible_host"]}\t(#{host.group_list.join ','})"
+    end
+  end
+
+  def keyscan_inventory
+    inventory.find_hosts!(@env || "all").each do |host|
+      ip = host.vars["ansible_host"]
+      system %Q(ssh-keygen -R #{ip})
+      system %Q(ssh-keyscan -Ht ed25519 #{ip} >> "$HOME/.ssh/known_hosts")
+    end
+  end
+
+  def capistrano_deployrb
+    if @env.nil?
+      puts "Please provide an environment e.g: subspace inventory capistrano --env production"
+      exit
+    end
+
+    say "# config/deploy/#{@env}.rb"
+    say "# Generated by Subspace"
+    inventory.find_hosts!(@env).each do |host|
+      host = inventory.hosts[host.name]
+      db_role = false
+      roles = host.group_list.map do |group_name|
+        if group_name =~ /web/
+          ["web", "app"]
+        elsif group_name =~ /worker/
+          ["app", db_role ? nil : "db"]
+        end
+      end.compact.uniq
+      say "server '#{host.vars["ansible_host"]}', user: 'deploy', roles: %w{#{roles.join(' ')}} # #{host.vars["hostname"]}"
+    end
+  end
+end

data/lib/subspace/commands/maintain.rb

@@ -11,6 +11,6 @@ class Subspace::Commands::Maintain < Subspace::Commands::Base
   def run
     ansible_options = ["--diff", "--tags=maintenance"]
     ansible_options = ansible_options | pass_through_params
-    ansible_command "ansible-playbook", "#{@environment}.yml", *ansible_options
+    ansible_playbook "#{@environment}.yml", *ansible_options
   end
 end

data/lib/subspace/commands/provision.rb

@@ -9,8 +9,6 @@ class Subspace::Commands::Provision < Subspace::Commands::Base
   end
 
   def run
-    ansible_options = ["--diff"]
-    ansible_options = ansible_options | pass_through_params
-    ansible_command "ansible-playbook", "#{@environment}.yml", *ansible_options
+    ansible_playbook "#{@environment}.yml", *pass_through_params
   end
 end

data/lib/subspace/commands/secrets.rb

@@ -0,0 +1,69 @@
+class Subspace::Commands::Secrets < Subspace::Commands::Base
+  def initialize(args, options)
+    if args.first == "rekey"
+      rekey
+    else
+      @environment = args.first
+      @action = if options.edit
+        "edit"
+      elsif options.create
+        "create"
+      else
+        "view"
+      end
+
+      run
+    end
+  end
+
+  def run
+    update_ansible_cfg
+    case @action
+    when "create"
+      create_local
+    when "view", "edit"
+      ansible_command "ansible-vault", @action, "secrets/#{@environment}.yml"
+    else
+      abort "Invalid secrets command"
+    end
+  end
+
+  def create_local
+    if File.exists? File.join(project_path, "config/application.yml")
+      answer = ask "config/application.yml already exists. Reply 'yes' to overwrite: [no] "
+      abort unless answer == "yes"
+    end
+    src = application_yml_template
+    dest = "config/application.yml"
+    vars_file = File.join(project_path, dest_dir, "/secrets/#{@environment}.yml")
+    extra_vars = "project_path=#{project_path} vars_file=#{vars_file} src=#{src} dest=#{dest}"
+    ansible_command "ansible-playbook", File.join(playbook_dir, "local_template.yml"), "--extra-vars", extra_vars
+    say "File created at config/application.yml with #{@environment} secrets"
+    say "-------------------------------------------------------------------\n"
+
+    system "cat", "config/application.yml"
+  end
+
+  def rekey
+    secret_files = Dir.glob("config/subspace/secrets/*.yml").map {|x| "secrets/#{File.basename(x)}"}
+    exit unless agree("This will re-key your secrets with a new random vault_pass. (#{secret_files}).  Proceed? (yes to continue) ")
+
+
+    say "Writing new password to .vault_pass.new"
+    File.write "config/subspace/.vault_pass.new", SecureRandom.base64(24) + "\n"
+    success = ansible_command "ansible-vault", "rekey", "--vault-password-file", ".vault_pass", "--new-vault-password-file", ".vault_pass.new", "-v", *secret_files
+    if success
+      FileUtils.mv "config/subspace/.vault_pass", "config/subspace/.vault_pass.old"
+      FileUtils.mv "config/subspace/.vault_pass.new", "config/subspace/.vault_pass"
+    else
+      say "Something went wrong, not changing .vault_pass"
+    end
+  end
+
+  private
+
+  def application_yml_template
+    "#{dest_dir}/templates/application.yml.template"
+  end
+
+end

data/lib/subspace/commands/ssh.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'subspace/inventory'
 class Subspace::Commands::Ssh < Subspace::Commands::Base
   PASS_THROUGH_PARAMS = ["i"]
 
@@ -10,18 +11,23 @@ class Subspace::Commands::Ssh < Subspace::Commands::Base
   end
 
   def run
-    if !File.exists? "config/provision/host_vars/#{@host}"
+    if !inventory.hosts[@host]
       say "No host '#{@host}' found. "
-      all_hosts = Dir["config/provision/host_vars/*"].collect {|f| File.basename(f) }
+      all_hosts = inventory.hosts.keys
       say (["Available hosts:"] + all_hosts).join("\n\t")
       return
     end
-    host_vars = YAML.load_file("config/provision/host_vars/#{@host}")
-    user = @user || host_vars["ansible_ssh_user"] || host_vars["ansible_user"]
-    host = host_vars["ansible_ssh_host"] || host_vars["ansible_host"]
-    port = host_vars["ansible_ssh_port"] || host_vars["ansible_port"] || 22
-    cmd = "ssh #{user}@#{host} -p #{port} #{pass_through_params.join(" ")}"
-    say cmd
+    host_vars = inventory.hosts[@host].vars
+    if host_vars.key?('ansible_ssh_user')
+      say "Supposed to be ansible_user not ansible_ssh_user"
+    end
+    user = @user || host_vars["ansible_user"]
+    host = host_vars["ansible_host"]
+    port = host_vars["ansible_port"] || 22
+    pem = host_vars["ansible_ssh_private_key_file"]
+    pem_cmd = "-i config/subspace/#{pem}" if pem
+    cmd = "ssh #{user}@#{host} -p #{port} #{pem_cmd} #{pass_through_params.join(" ")}"
+    say "> #{cmd} \n"
     exec cmd
   end
 end

data/lib/subspace/commands/terraform.rb

@@ -0,0 +1,83 @@
+require 'json'
+module Subspace
+  module Commands
+    class Terraform < Subspace::Commands::Base
+
+      def self.check_aws_credentials(project_name)
+        ENV["AWS_ACCESS_KEY_ID"] = nil
+        ENV["AWS_SECRET_ACCESS_KEY"] = nil
+
+        profile = "subspace-#{project_name}"
+
+        system("aws --profile #{profile} configure list &> /dev/null ")
+        if $? != 0
+          puts "No AWS Profile '#{profile}' configured.  Please enter your credentials."
+          system("aws --profile #{profile} configure")
+          system("aws --profile #{profile} configure list &> /dev/null ")
+          if $? != 0
+            puts "FATAL: could not configure aws.  Please try again"
+            exit
+          end
+        else
+          puts "Using AWS Profile #{profile}"
+        end
+        true
+      end
+
+      def self.ensure_terraform
+        if `terraform -v --json | jq -r .terraform_version` =~ /1\.\d+/
+          puts "Terraform found."
+          return true
+        else
+          puts "Please install terraform at least 1.1 locally"
+          return false
+        end
+      end
+
+      def initialize(args, options)
+        @env = args.shift
+        @args = args
+        @options = options
+        run
+      end
+
+      def run
+        self.class.check_aws_credentials(project_name) or exit
+        self.class.ensure_terraform or exit
+        if @args.any?
+          terraform_command(@args.shift, *@args)
+        else
+          puts "No command specified, running plan/apply"
+          terraform_command("init", "-upgrade") or return
+          terraform_command("apply") or return
+          update_inventory
+        end
+      end
+
+      private
+      def terraform_command(command, *args)
+        result = nil
+        # update_terraformrc
+        Dir.chdir "config/subspace/terraform/#{@env}" do
+          say ">> Running terraform #{command} #{args.join(' ')}"
+          result = system("terraform", command, *args, out: $stdout, err: $stderr)
+          say "<< Done"
+        end
+        result
+      end
+
+      def update_terraformrc
+        template! "terraformrc", ".terraformrc"
+      end
+
+      def update_inventory
+        puts "Apply succeeded, updating inventory."
+        Dir.chdir "config/subspace/terraform/#{@env}" do
+          @output = JSON.parse `terraform output -json inventory`
+        end
+        inventory.merge(@output)
+        inventory.write
+      end
+    end
+  end
+end