subspace 2.5.7 → 3.0.0.rc1

data/lib/subspace/commands/init.rb

@@ -3,76 +3,113 @@ require 'erb'
 require 'securerandom'
 class Subspace::Commands::Init < Subspace::Commands::Base
   def initialize(args, options)
-    if args.first == "vars"
-      init_vars
-    else
-      run
+    options.default env: "dev"
+
+    @env = options.env
+    @template = options.template
+
+    if options.ansibe.nil? && options.terraform.nil?
+      # They didn't pass in any options (subspace init) so just do both
+      options.ansible = true
+      options.terraform = true
+    end
+
+    if @template.nil? && options.terraform == true
+      answer = ask "What template/server configuration would you like to use? e.g. 'workhorse' or 'oxenwagen'"
+      @template = answer
     end
+
+    @init_ansible = options.ansible
+    @init_terraform = options.terraform
+    run
   end
 
   def run
     if File.exists? dest_dir
       answer = ask "Subspace appears to be initialized.  Reply 'yes' to continue anyway: [no] "
       abort unless answer.chomp == "yes"
+    else
+      FileUtils.mkdir_p dest_dir
     end
 
-    FileUtils.mkdir_p File.join dest_dir, "group_vars"
-    FileUtils.mkdir_p File.join dest_dir, "host_vars"
-    FileUtils.mkdir_p File.join dest_dir, "vars"
-    FileUtils.mkdir_p File.join dest_dir, "roles"
+    init_pemfile
+    init_ansible if @init_ansible
+    init_terraform if @init_terraform
 
-    copy ".gitignore"
-    #template "../provision.rb"
-    template "ansible.cfg"
-    template "hosts"
-    template "group_vars/all"
+    puts """
+    1. Inspect key config files:
+      - config/subspace/terraform/#{@env}/main.tf     # Main terraform file
+      - config/subspace/#{@env}.yml                   # Main ansible playbook
+      - config/subspace/group_vars                    # Ansible configuration options
+      - config/subspace/inventory.yml                 # Server Inventory
+      - config/subspace/templates/authorized_keys     # SSH Authorized Keys
+      - config/subspace/templates/application.yml     # Application Environment variables
 
-    create_vault_pass
-    environments.each do |env|
-      @env = env
-      @hostname = hostname(env)
-      template "group_vars/template", "group_vars/#{env}"
-      template "host_vars/template", "host_vars/#{env}"
-      create_vars_file_for_env env
-      template "playbook.yml", "#{env}.yml"
-    end
-    create_vars_file_for_env "development"
-    init_vars
+    2. create cloud infrastructure with terraform:
 
-    puts """
-    1. Create a server.
+      subspace tf #{@env}
 
-    2. Set your server's location:
+    3. Bootstrap the new server
 
-      vim config/provision/host_vars/production
-      vim config/provision/host_vars/dev
+      subspace boostrap #{@env}1
 
-    3. Set up your authorized_keys:
+    4. Inspect new environment
+      - ensure the correct roles are present in #{@env}.yml
+      - Check ansible configuration variables in group_vars/#{@env}
 
-      vim config/provision/authorized_keys
+    4. Provision the new servers with ansible:
 
-    4. Then provision your server:
+      subspace provision #{@env}
 
-      subspace provision dev
-      subspace provision production
+    !!MAKE SURE YOU PUT config/subspace/subspace.pem SOMEWHERE!!
 
   """
 
   end
 
-  def init_vars
-    FileUtils.mkdir_p File.join dest_dir, "templates"
-    copy "templates/application.yml.template"
+  private
+
+  def init_pemfile
+    pem = File.join dest_dir, "subspace.pem"
+    if File.exist?(pem)
+      say "Existing SSH Keypair exists.  Skipping keygen."
+    else
+      say "Creating SSH Keypair in #{pem}"
+      `ssh-keygen -t rsa -f #{pem}`
+    end
   end
 
-  private
+  def init_ansible
+    FileUtils.mkdir_p File.join dest_dir, "group_vars"
+    FileUtils.mkdir_p File.join dest_dir, "secrets"
+    FileUtils.mkdir_p File.join dest_dir, "roles"
+    FileUtils.mkdir_p File.join dest_dir, "templates"
+
+    copy ".gitignore"
+    template "ansible.cfg"
+    template "group_vars/all"
+    template "inventory.yml"
+    template "templates/authorized_keys"
 
-  def project_name
-    File.basename(Dir.pwd)
+    create_vault_pass
+    @hostname = hostname(@env)
+    create_secrets_for @env
+    template "group_vars/template", "group_vars/#{@env}"
+    template "playbook.yml", "#{@env}.yml"
+
+    create_secrets_for "development" #TODO rename to local?
+    init_appyml
   end
 
-  def environments
-    %w(production dev)
+  def init_terraform
+    Subspace::Commands::Terraform.ensure_terraform
+    Subspace::Commands::Terraform.check_aws_credentials(project_name)
+
+    FileUtils.mkdir_p File.join dest_dir, "terraform", @env
+
+    set_latest_ami
+    template "terraform/template/main-#{@template}.tf", "terraform/#{@env}/main.tf"
+    copy "terraform/.gitignore"
   end
 
   def hostname(env)
@@ -87,10 +124,22 @@ class Subspace::Commands::Init < Subspace::Commands::Base
     end
   end
 
-  def create_vars_file_for_env(env)
-    template "vars/template", "vars/#{env}.yml"
+  def create_secrets_for(env)
+    template "secrets/template", "secrets/#{env}.yml"
     Dir.chdir dest_dir do
-      `ansible-vault encrypt vars/#{env}.yml`
+      `ansible-vault encrypt secrets/#{env}.yml`
     end
   end
+
+  def init_appyml
+    copy "templates/application.yml.template"
+  end
+
+  def set_latest_ami
+    @latest_ami = `aws --profile subspace-#{project_name} ec2 describe-images \
+    --filters 'Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64*' \
+    --query 'Images[*].[ImageId,CreationDate]' --output text \
+    | sort -k2 -r \
+    | head -n1 | cut -f1`.chomp
+  end
 end

data/lib/subspace/commands/inventory.rb

@@ -0,0 +1,45 @@
+class Subspace::Commands::Inventory < Subspace::Commands::Base
+
+  def initialize(args, options)
+    command = args.first
+    @env = options.env
+    case command
+    when "capistrano"
+      capistrano_deployrb
+    when "list"
+      list_inventory
+    else
+      say "Unknown or missing command to inventory: #{command}"
+      say "try subspace inventory [list, capistrano]"
+    end
+  end
+
+  def list_inventory
+    inventory.find_hosts!(@env || "all").each do |host_name|
+      host = inventory.hosts[host_name]
+      puts "#{host.name}\t#{host.vars["ansible_host"]}\t(#{host.group_list.join ','})"
+    end
+  end
+
+  def capistrano_deployrb
+    if @env.nil?
+      puts "Please provide an environment e.g: subspace inventory capistrano --env production"
+      exit
+    end
+
+    say "# config/deploy/#{@env}.rb"
+    say "# Generated by Subspace"
+    inventory.find_hosts!(@env).each do |host_name|
+      host = inventory.hosts[host_name]
+      db_role = false
+      roles = host.group_list.map do |group_name|
+        if group_name =~ /web/
+          ["web", "app"]
+        elsif group_name =~ /worker/
+          ["app", db_role ? nil : "db"]
+        end
+      end.compact.uniq
+      say "server '#{host.vars["ansible_host"]}', user: 'deploy', roles: %w{#{roles.join(' ')}} # #{host.vars["hostname"]}"
+    end
+  end
+end

data/lib/subspace/commands/maintain.rb

@@ -11,6 +11,6 @@ class Subspace::Commands::Maintain < Subspace::Commands::Base
   def run
     ansible_options = ["--diff", "--tags=maintenance"]
     ansible_options = ansible_options | pass_through_params
-    ansible_command "ansible-playbook", "#{@environment}.yml", *ansible_options
+    ansible_playbook "#{@environment}.yml", *ansible_options
   end
 end

data/lib/subspace/commands/provision.rb

@@ -9,8 +9,6 @@ class Subspace::Commands::Provision < Subspace::Commands::Base
   end
 
   def run
-    ansible_options = ["--diff"]
-    ansible_options = ansible_options | pass_through_params
-    ansible_command "ansible-playbook", "#{@environment}.yml", *ansible_options
+    ansible_playbook "#{@environment}.yml", *pass_through_params
   end
 end

data/lib/subspace/commands/{vars.rb → secrets.rb}

@@ -1,4 +1,4 @@
-class Subspace::Commands::Vars < Subspace::Commands::Base
+class Subspace::Commands::Secrets < Subspace::Commands::Base
   def initialize(args, options)
     @environment = args.first
     @action = if options.edit
@@ -13,13 +13,14 @@ class Subspace::Commands::Vars < Subspace::Commands::Base
   end
 
   def run
+    update_ansible_cfg
     case @action
     when "create"
       create_local
     when "view", "edit"
-      ansible_command "ansible-vault", @action, "vars/#{@environment}.yml"
+      ansible_command "ansible-vault", @action, "secrets/#{@environment}.yml"
     else
-      abort "Invalid vars command"
+      abort "Invalid secrets command"
     end
   end
 
@@ -30,7 +31,7 @@ class Subspace::Commands::Vars < Subspace::Commands::Base
     end
     src = application_yml_template
     dest = "config/application.yml"
-    vars_file = File.join(project_path, "config/provision/vars/#{@environment}.yml")
+    vars_file = File.join(project_path, dest_dir, "/secrets/#{@environment}.yml")
     extra_vars = "project_path=#{project_path} vars_file=#{vars_file} src=#{src} dest=#{dest}"
     ansible_command "ansible-playbook", File.join(playbook_dir, "local_template.yml"), "--extra-vars", extra_vars
     say "File created at config/application.yml with #{@environment} secrets"
@@ -42,7 +43,7 @@ class Subspace::Commands::Vars < Subspace::Commands::Base
   private
 
   def application_yml_template
-    "config/provision/templates/application.yml.template"
+    "#{dest_dir}/templates/application.yml.template"
   end
 
 end

data/lib/subspace/commands/ssh.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'subspace/inventory'
 class Subspace::Commands::Ssh < Subspace::Commands::Base
   PASS_THROUGH_PARAMS = ["i"]
 
@@ -10,18 +11,19 @@ class Subspace::Commands::Ssh < Subspace::Commands::Base
   end
 
   def run
-    if !File.exists? "config/provision/host_vars/#{@host}"
+    if !inventory.hosts[@host]
       say "No host '#{@host}' found. "
-      all_hosts = Dir["config/provision/host_vars/*"].collect {|f| File.basename(f) }
+      all_hosts = inventory.hosts.keys
       say (["Available hosts:"] + all_hosts).join("\n\t")
       return
     end
-    host_vars = YAML.load_file("config/provision/host_vars/#{@host}")
-    user = @user || host_vars["ansible_ssh_user"] || host_vars["ansible_user"]
-    host = host_vars["ansible_ssh_host"] || host_vars["ansible_host"]
-    port = host_vars["ansible_ssh_port"] || host_vars["ansible_port"] || 22
-    cmd = "ssh #{user}@#{host} -p #{port} #{pass_through_params.join(" ")}"
-    say cmd
+    host_vars = inventory.hosts[@host].vars
+    user = host_vars["ansible_user"]
+    host = host_vars["ansible_host"]
+    port = host_vars["ansible_port"] || 22
+    pem = host_vars["ansible_ssh_private_key_file"] || 'subspace.pem'
+    cmd = "ssh #{user}@#{host} -p #{port} -i config/subspace/#{pem} #{pass_through_params.join(" ")}"
+    say "> #{cmd} \n"
     exec cmd
   end
 end

data/lib/subspace/commands/terraform.rb

@@ -0,0 +1,83 @@
+require 'json'
+module Subspace
+  module Commands
+    class Terraform < Subspace::Commands::Base
+
+      def self.check_aws_credentials(project_name)
+        ENV["AWS_ACCESS_KEY_ID"] = nil
+        ENV["AWS_SECRET_ACCESS_KEY"] = nil
+
+        profile = "subspace-#{project_name}"
+
+        system("aws --profile #{profile} configure list &> /dev/null ")
+        if $? != 0
+          puts "No AWS Profile '#{profile}' configured.  Please enter your credentials."
+          system("aws --profile #{profile} configure")
+          system("aws --profile #{profile} configure list &> /dev/null ")
+          if $? != 0
+            puts "FATAL: could not configure aws.  Please try again"
+            exit
+          end
+        else
+          puts "Using AWS Profile #{profile}"
+        end
+        true
+      end
+
+      def self.ensure_terraform
+        if `terraform -v --json | jq -r .terraform_version` =~ /1\.\d+/
+          puts "Terraform found."
+          return true
+        else
+          puts "Please install terraform at least 1.1 locally"
+          return false
+        end
+      end
+
+      def initialize(args, options)
+        @env = args.shift
+        @args = args
+        @options = options
+        run
+      end
+
+      def run
+        self.class.check_aws_credentials(project_name) or exit
+        self.class.ensure_terraform or exit
+        if @args.any?
+          terraform_command(@args.shift, *@args)
+        else
+          puts "No command specified, running plan/apply"
+          terraform_command("init", "-upgrade") or return
+          terraform_command("apply") or return
+          update_inventory
+        end
+      end
+
+      private
+      def terraform_command(command, *args)
+        result = nil
+        # update_terraformrc
+        Dir.chdir "config/subspace/terraform/#{@env}" do
+          say ">> Running terraform #{command} #{args.join(' ')}"
+          result = system("terraform", command, *args, out: $stdout, err: $stderr)
+          say "<< Done"
+        end
+        result
+      end
+
+      def update_terraformrc
+        template! "terraformrc", ".terraformrc"
+      end
+
+      def update_inventory
+        puts "Apply succeeded, updating inventory."
+        Dir.chdir "config/subspace/terraform/#{@env}" do
+          @output = JSON.parse `terraform output -json oxenwagen`
+        end
+        inventory.merge(@output)
+        inventory.write
+      end
+    end
+  end
+end

data/lib/subspace/inventory.rb

@@ -0,0 +1,144 @@
+require 'yaml'
+module Subspace
+  class Inventory
+    attr_accessor :group_vars, :hosts, :global_vars, :path
+    def initialize
+      @hosts = {}
+      @group_vars = {}
+      @global_vars = {}
+    end
+
+    # Find all the hosts in the host/group or exit
+    def find_hosts!(host_spec)
+      if self.groups[host_spec]
+        return self.groups[host_spec].host_list.map { |m| self.hosts[m] }
+      elsif self.hosts[host_spec]
+        return [self.hosts[host_spec]]
+      else
+        say "No inventory matching: '#{host_spec}' found. "
+        say (["Available hosts:"] + self.hosts.keys).join("\n\t")
+        say (["Available groups:"] + self.groups.keys).join("\n\t")
+        exit
+      end
+    end
+
+    def self.read(path)
+      inventory = new
+      inventory.path = path
+
+      yml = YAML.load(File.read(path)).to_h
+
+      # Run through all hosts
+      yml["all"]["hosts"].each do |name, vars|
+        inventory.hosts[name] = Host.new(name, vars: vars || {})
+      end
+
+      # Run through all children (groups)
+      # This does NOT handle sub-groups yet
+      yml["all"]["children"].each do |name, group|
+        next unless group["hosts"]
+
+        # Each group defines its host membership
+        group["hosts"].each do |host, vars|
+          inventory.hosts[host] ||= Host.new(host, vars: vars || {})
+          inventory.hosts[host].group_list.push name
+        end
+
+        if group["vars"]
+          inventory.group_vars[name] = group["vars"]
+        end
+      end
+
+      # Capture global variables
+      inventory.global_vars = yml["all"]["vars"] || {}
+
+      inventory
+    end
+
+    def write(path=nil)
+      File.write(@path || path, self.to_yml)
+    end
+
+    def merge(inventory_json)
+      inventory_json["inventory"]["hostnames"].each_with_index do |host, i|
+        if hosts[host]
+          old_ip = hosts[host].vars["ansible_host"]
+          new_ip = inventory_json["inventory"]["ip_addresses"][i]
+          if old_ip != new_ip
+            say "  * Host '#{host}' IP address changed! You may need to update the inventory! (#{old_ip} => #{new_ip})"
+          end
+          next
+        end
+        hosts[host] = Host.new(host)
+        hosts[host].vars["ansible_host"] = inventory_json["inventory"]["ip_addresses"][i]
+        hosts[host].vars["ansible_user"] = inventory_json["inventory"]["users"][i]
+        hosts[host].vars["hostname"] = host
+        hosts[host].group_list = inventory_json["inventory"]["groups"][i].split(/\s/)
+      end
+    end
+
+    def to_yml
+      all_groups = {}
+      all_hosts = {}
+      @hosts.each do |name, host|
+        all_hosts[host.name] = host.vars.empty? ? nil : host.vars.transform_keys(&:to_s)
+        host.group_list.each do |group|
+          all_groups[group] ||= { "hosts" => {}}
+          all_groups[group]["hosts"][host.name] = nil
+        end
+      end
+
+      @group_vars.each do |group, vars|
+        all_groups[group] ||= {}
+        if !vars.empty?
+          all_groups[group]["vars"] = vars.transform_keys(&:to_s)
+        end
+      end
+
+      yml = {
+        "all" => {
+          "hosts" => all_hosts,
+          "children" => all_groups.empty? ? nil : all_groups
+        }
+      }
+
+      if !@global_vars.empty?
+        yml["all"]["vars"] = @global_vars.transform_keys(&:to_s)
+      end
+
+      YAML.dump(yml)
+    end
+
+    def groups
+      @groups ||= begin
+        all_groups = {"all" => Group.new("all", vars: {}, host_list: hosts.keys) }
+        @hosts.each do |name, host|
+          host.group_list.each do |group|
+            all_groups[group] ||= Group.new(group, vars: @group_vars[group])
+            all_groups[group].host_list.append(name)
+          end
+        end
+        all_groups
+      end
+    end
+
+    class Host
+      attr_accessor :vars, :group_list, :name
+
+      def initialize(name, vars: {}, group_list: [])
+        @name = name
+        @vars = vars
+        @group_list = group_list
+      end
+    end
+
+    class Group
+      attr_accessor :name, :vars, :host_list
+      def initialize(name, vars: {}, host_list: [])
+        @name = name
+        @vars = vars
+        @host_list = host_list
+      end
+    end
+  end
+end

data/lib/subspace/version.rb

@@ -1,3 +1,3 @@
 module Subspace
-  VERSION = "2.5.7"
+  VERSION = "3.0.0.rc1"
 end

data/subspace.gemspec

@@ -13,6 +13,13 @@ Gem::Specification.new do |spec|
   spec.description   = %q{WIP -- don't use this :)}
   spec.homepage      = "https://github.com/tenforwardconsulting/subspace"
   spec.license       = "MIT"
+  spec.post_install_message = <<~EOS
+    *** Subspace 3 has many breaking changes
+    Primarily, the entire configuration directory structure has moved from config/provision to config/subspace.
+    You will need to migrate your old configuration to the new location, or downgrade to Subspace 2 if this was not intentional.
+    Please review the Upgrade guide: https://github.com/tenforwardconsulting/subspace/UPGRADING.md
+    EOS
+
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -24,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   << "subspace"
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.1"

data/template/{provision → subspace}/.gitignore

@@ -1,3 +1,6 @@
 .vault_pass
 ansible.cfg
 *.retry
+*.tfstate*
+*.pem
+.terraform

data/template/subspace/ansible.cfg.erb

@@ -0,0 +1,16 @@
+[defaults]
+forks     = 10
+roles_path = ./roles:<%= File.join(gem_path, 'ansible', 'roles') %>:/etc/ansible/roles
+vault_password_file = .vault_pass
+inventory = inventory.yml
+# Uncomment to add timestamps to tasks to find slow ones.
+# callback_whitelist = profile_tasks
+
+<% if @mitogen_path %>
+strategy_plugins = <%= @mitogen_path %>/ansible_mitogen/plugins/strategy
+strategy = mitogen_linear
+<% end %>
+
+[ssh_connection]
+pipelining = True
+control_path = /tmp/subspace-control-%%h-%%p-%%r

data/template/subspace/group_vars/all.erb

@@ -0,0 +1,28 @@
+# Conventions and defaults are defined here
+project_name: <%= project_name %>
+database_host: localhost
+use_sudo: true
+
+# ruby-common
+# pull the checksum/url from https://www.ruby-lang.org/en/downloads/
+ruby_version: # ruby-2.7.1
+ruby_checksum: # d418483bdd0000576c1370571121a6eb24582116db0b7bb2005e90e250eae418
+ruby_download_location: # https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.1.tar.gz
+bundler_version: # 2.3.5
+
+# Other stuff
+letsencrypt_email:
+nodejs_version:  # 16.x
+ssl_enabled: true
+postgresql_version: # 14
+
+logrotate_scripts:
+  - name: rails
+    path: "/u/apps/{{project_name}}/shared/log/{{rails_env}}.log"
+    options:
+      - weekly
+      - size 100M
+      - missingok
+      - compress
+      - delaycompress
+      - copytruncate

data/template/subspace/group_vars/template.erb

@@ -0,0 +1,26 @@
+## rails
+rails_env: <%= @env %>
+
+database_pool: 5
+database_name: "{{project_name}}_{{rails_env}}"
+database_user: "{{project_name}}"
+database_adapter: postgresql
+# job_queues:
+#   - default
+#   - mailers
+
+# nginx / letsencrypt
+server_name: "{{ansible_host}}"
+
+## postgresql
+# postgresql_version: 14
+# postgresql_authentication:
+#   - type: local
+#     user: "{{database_user}}"
+#     database: 'all'
+#     method: trust
+
+## puma
+puma_workers: 1
+puma_min_threads: 4
+puma_max_threads: 16

data/template/subspace/inventory.yml.erb

@@ -0,0 +1,11 @@
+---
+all:
+  hosts:
+    <%= @env %>1:
+      ansible_host: localhost
+      ansible_user: ubuntu
+      ansible_ssh_private_key_file: subspace.pem
+      hostname: web1
+  children:
+    <%= @env %>:
+      <%= @env %>1: