subspace 2.5.10 → 3.0.0

data/lib/subspace/inventory.rb

@@ -0,0 +1,144 @@
+require 'yaml'
+module Subspace
+  class Inventory
+    attr_accessor :group_vars, :hosts, :global_vars, :path
+    def initialize
+      @hosts = {}
+      @group_vars = {}
+      @global_vars = {}
+    end
+
+    # Find all the hosts in the host/group or exit
+    def find_hosts!(host_spec)
+      if self.groups[host_spec]
+        return self.groups[host_spec].host_list.map { |m| self.hosts[m] }
+      elsif self.hosts[host_spec]
+        return [self.hosts[host_spec]]
+      else
+        say "No inventory matching: '#{host_spec}' found. "
+        say (["Available hosts:"] + self.hosts.keys).join("\n\t")
+        say (["Available groups:"] + self.groups.keys).join("\n\t")
+        exit
+      end
+    end
+
+    def self.read(path)
+      inventory = new
+      inventory.path = path
+
+      yml = YAML.load(File.read(path)).to_h
+
+      # Run through all hosts
+      yml["all"]["hosts"].each do |name, vars|
+        inventory.hosts[name] = Host.new(name, vars: vars || {})
+      end
+
+      # Run through all children (groups)
+      # This does NOT handle sub-groups yet
+      yml["all"]["children"].each do |name, group|
+        next unless group["hosts"]
+
+        # Each group defines its host membership
+        group["hosts"].each do |host, vars|
+          inventory.hosts[host] ||= Host.new(host, vars: vars || {})
+          inventory.hosts[host].group_list.push name
+        end
+
+        if group["vars"]
+          inventory.group_vars[name] = group["vars"]
+        end
+      end
+
+      # Capture global variables
+      inventory.global_vars = yml["all"]["vars"] || {}
+
+      inventory
+    end
+
+    def write(path=nil)
+      File.write(@path || path, self.to_yml)
+    end
+
+    def merge(inventory_json)
+      inventory_json["hostnames"].each_with_index do |host, i|
+        if hosts[host]
+          old_ip = hosts[host].vars["ansible_host"]
+          new_ip = inventory_json["ip_addresses"][i]
+          if old_ip != new_ip
+            say "  * Host '#{host}' IP address changed! You may need to update the inventory! (#{old_ip} => #{new_ip})"
+          end
+          next
+        end
+        hosts[host] = Host.new(host)
+        hosts[host].vars["ansible_host"] = inventory_json["ip_addresses"][i]
+        hosts[host].vars["ansible_user"] = inventory_json["users"][i]
+        hosts[host].vars["hostname"] = host
+        hosts[host].group_list = inventory_json["groups"][i].split(/\s/)
+      end
+    end
+
+    def to_yml
+      all_groups = {}
+      all_hosts = {}
+      @hosts.each do |name, host|
+        all_hosts[host.name] = host.vars.empty? ? nil : host.vars.transform_keys(&:to_s)
+        host.group_list.each do |group|
+          all_groups[group] ||= { "hosts" => {}}
+          all_groups[group]["hosts"][host.name] = nil
+        end
+      end
+
+      @group_vars.each do |group, vars|
+        all_groups[group] ||= {}
+        if !vars.empty?
+          all_groups[group]["vars"] = vars.transform_keys(&:to_s)
+        end
+      end
+
+      yml = {
+        "all" => {
+          "hosts" => all_hosts,
+          "children" => all_groups.empty? ? nil : all_groups
+        }
+      }
+
+      if !@global_vars.empty?
+        yml["all"]["vars"] = @global_vars.transform_keys(&:to_s)
+      end
+
+      YAML.dump(yml)
+    end
+
+    def groups
+      @groups ||= begin
+        all_groups = {"all" => Group.new("all", vars: {}, host_list: hosts.keys) }
+        @hosts.each do |name, host|
+          host.group_list.each do |group|
+            all_groups[group] ||= Group.new(group, vars: @group_vars[group])
+            all_groups[group].host_list.append(name)
+          end
+        end
+        all_groups
+      end
+    end
+
+    class Host
+      attr_accessor :vars, :group_list, :name
+
+      def initialize(name, vars: {}, group_list: [])
+        @name = name
+        @vars = vars
+        @group_list = group_list
+      end
+    end
+
+    class Group
+      attr_accessor :name, :vars, :host_list
+      def initialize(name, vars: {}, host_list: [])
+        @name = name
+        @vars = vars
+        @host_list = host_list
+      end
+    end
+  end
+end

data/lib/subspace/version.rb

@@ -1,3 +1,3 @@
 module Subspace
-  VERSION = "2.5.10"
+  VERSION = "3.0.0"
 end

data/subspace.gemspec

@@ -13,6 +13,13 @@ Gem::Specification.new do |spec|
   spec.description   = %q{WIP -- don't use this :)}
   spec.homepage      = "https://github.com/tenforwardconsulting/subspace"
   spec.license       = "MIT"
+  spec.post_install_message = <<~EOS
+    *** Subspace 3 has many breaking changes
+    Primarily, the entire configuration directory structure has moved from config/provision to config/subspace.
+    You will need to migrate your old configuration to the new location, or downgrade to Subspace 2 if this was not intentional.
+    Please review the Upgrade guide: https://github.com/tenforwardconsulting/subspace/UPGRADING.md
+    EOS
+
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
@@ -24,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.executables   << "subspace"
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.1"
@@ -33,5 +40,4 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "commander", "~>4.2"
   spec.add_runtime_dependency "figaro", "~>1.0"
-  spec.add_runtime_dependency "ed25519", "~>1.0"
 end

data/template/{provision → subspace}/.gitignore

@@ -1,3 +1,6 @@
 .vault_pass
 ansible.cfg
 *.retry
+*.tfstate*
+*.pem
+.terraform

data/template/{provision → subspace}/ansible.cfg.erb

@@ -1,8 +1,8 @@
 [defaults]
-inventory = hosts
 forks     = 10
 roles_path = ./roles:<%= File.join(gem_path, 'ansible', 'roles') %>:/etc/ansible/roles
 vault_password_file = .vault_pass
+inventory = inventory.yml
 # Uncomment to add timestamps to tasks to find slow ones.
 # callback_whitelist = profile_tasks
 
@@ -13,4 +13,4 @@ strategy = mitogen_linear
 
 [ssh_connection]
 pipelining = True
-control_path = /tmp/subspace-control-%%h-%%p-%%r
+control_path = /tmp/subspace-control-%%h-%%p-%%r

data/template/subspace/group_vars/all.erb

@@ -0,0 +1,28 @@
+# Conventions and defaults are defined here
+project_name: <%= project_name %>
+database_host: localhost
+use_sudo: true
+
+# ruby-common
+# pull the checksum/url from https://www.ruby-lang.org/en/downloads/
+ruby_version: # ruby-2.7.1
+ruby_checksum: # d418483bdd0000576c1370571121a6eb24582116db0b7bb2005e90e250eae418
+ruby_download_location: # https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.1.tar.gz
+bundler_version: # 2.3.5
+
+# Other stuff
+letsencrypt_email:
+nodejs_version:  # 16.x
+ssl_enabled: true
+postgresql_version: # 14
+
+logrotate_scripts:
+  - name: rails
+    path: "/u/apps/{{project_name}}/shared/log/{{rails_env}}.log"
+    options:
+      - weekly
+      - size 100M
+      - missingok
+      - compress
+      - delaycompress
+      - copytruncate

data/template/subspace/group_vars/template.erb

@@ -0,0 +1,26 @@
+## rails
+rails_env: <%= @env %>
+
+database_pool: 5
+database_name: "{{project_name}}_{{rails_env}}"
+database_user: "{{project_name}}"
+database_adapter: postgresql
+# job_queues:
+#   - default
+#   - mailers
+
+# nginx / letsencrypt
+server_name: "{{ansible_host}}"
+
+## postgresql
+# postgresql_version: 14
+# postgresql_authentication:
+#   - type: local
+#     user: "{{database_user}}"
+#     database: 'all'
+#     method: trust
+
+## puma
+puma_workers: 1
+puma_min_threads: 4
+puma_max_threads: 16

data/template/subspace/inventory.yml.erb

@@ -0,0 +1,11 @@
+---
+all:
+  hosts:
+    <%= @env %>1:
+      ansible_host: localhost
+      ansible_user: ubuntu
+      ansible_ssh_private_key_file: subspace.pem
+      hostname: web1
+  children:
+    <%= @env %>:
+      <%= @env %>1:

data/template/{provision → subspace}/playbook.yml.erb

@@ -3,19 +3,16 @@
     become: yes
 
     vars_files:
-      - ./vars/<%= @env %>.yml
+      - ./secrets/<%= @env %>.yml
 
     roles:
       - common
-      - yarn
       - nodejs
+      - yarn
       - ruby-common
       - rails
       - puma
       - letsencrypt
       - nginx
       - postgresql
-      - monit
       - logrotate
-      - collectd
-      - delayed_job

data/template/subspace/templates/authorized_keys.erb

@@ -0,0 +1 @@
+<%= `cat $HOME/.ssh/id_rsa.pub` %>

data/template/subspace/terraform/.gitignore

@@ -0,0 +1,2 @@
+credentials.auto.tfvars
+.subspace-tf-modules

data/template/subspace/terraform/template/main-oxenwagen.tf.erb

@@ -0,0 +1,116 @@
+terraform {
+  # Default backend is just local.
+
+  # Uncomment to use s3
+  # backend "s3" {
+  #   bucket = "subspace-backend-<%= project_name %>"
+  #   key    = "subspace.<%= @env %>.tfstate"
+  #   region = "us-west-2"
+  # }
+
+  # Uncomment to use Terraform Cloud
+  # cloud {
+  #   organization = "<%= project_name %>"
+  #
+  #   workspaces {
+  #     name = "<%= @env %>"
+  #   }
+  # }
+
+}
+
+provider aws {
+  region = "us-west-2"
+  profile = "subspace-<%= project_name %>"
+  default_tags {
+    tags = {
+      Environment = "<%= @env %>"
+      Project     = "<%= project_name %>"
+    }
+  }
+}
+
+variable database_password { type = string }
+
+module oxenwagen {
+  source = "github.com/tenforwardconsulting/terraform-subspace-oxenwagen?ref=v2.1.0"
+  project_name = "<%= project_name %>"
+  project_environment =  "<%= @env %>"
+  aws_region = ## "us-west-2"
+  lb_health_check_path = "/"
+  subspace_public_key = file("../../subspace.pem.pub")
+
+  # Ubuntu Server 20.04 LTS (HVM), SSD Volume Type
+  instance_ami = "<%= @latest_ami %>"
+  web_instance_type = "t3.small"
+  web_instance_count = 2
+  worker_instance_type = "t3.medium"
+  worker_instance_count = 1
+  worker_volume_size = 100
+  ssh_cidr_blocks = [] # Put office/local/vpn IP addresses here
+
+  database_engine = "postgres"
+  database_engine_version = ## "14.1"
+  database_instance_class = "db.t3.medium"
+  database_name = "<%= "#{project_name}_#{@env}" %>"
+  database_username = "<%= project_name %>"
+  database_password = var.database_password
+  database_allocated_storage = 100
+  database_max_allocated_storage = 1000
+  database_iops = 1000
+
+  # lb_domain_name = "www.<%= project_name %>.com"
+  # lb_alternate_names = []
+}
+
+output "inventory" {
+  value = module.oxenwagen
+}
+
+resource "aws_s3_bucket" "bucket" {
+  bucket               = "<%= "#{project_name}-#{@env}-assets" %>"
+  acl    = "private"
+}
+
+resource "aws_s3_bucket_public_access_block" "block_public_acls" {
+  bucket                  = aws_s3_bucket.bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_iam_user" "s3_user" {
+  name = "<%= "#{project_name}-#{@env}-assets" %>-subspace-s3-user"
+}
+
+resource "aws_iam_user_policy" "s3-upload" {
+  name = "test"
+  user = aws_iam_user.s3_user.name
+
+  policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+            "s3:PutObject",
+            "s3:PutObjectAcl",
+            "s3:GetObject",
+            "s3:GetObjectVersion",
+            "s3:GetBucketAcl",
+            "s3:DeleteObject",
+            "s3:DeleteObjectVersion"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::<%= "#{project_name}-#{@env}-assets" %>",
+                "arn:aws:s3:::<%= "#{project_name}-#{@env}-assets" %>/*"
+            ]
+        }
+    ]
+}
+EOF
+}
+

data/template/subspace/terraform/template/main-workhorse.tf.erb

@@ -0,0 +1,41 @@
+terraform {
+  # Default backend is just local.
+
+  # Uncomment to use s3
+  # backend "s3" {
+  #   bucket = "subspace-backend-<%= project_name %>"
+  #   key    = "subspace.<%= @env %>.tfstate"
+  #   region = "us-west-2"
+  # }
+
+  # Uncomment to use Terraform Cloud
+  # cloud {
+  #   organization = "<%= project_name %>"
+  #
+  #   workspaces {
+  #     name = "<%= @env %>"
+  #   }
+  # }
+
+}
+
+module workhorse {
+  source = "github.com/tenforwardconsulting/terraform-subspace-workhorse?ref=v1.0.0"
+  project_name = "<%= project_name %>"
+  project_environment = "<%= @env %>"
+  aws_region = "us-west-2"
+  subspace_public_key = file("../../subspace.pem.pub")
+  # zone_id = "ZOJ6811VRVYBT" # 10fw.net
+  # subdomain = "<%= project_name.gsub("_", "-") %>"
+
+  # Ubuntu Server 20.04 LTS (HVM), SSD Volume Type
+  instance_ami = "ami-0f81e6e71078b75b6"
+  instance_user = "ubuntu"
+  instance_type = "t3.medium"
+  instance_hostname = "${var.project_environment}-app1"
+  instance_volume_size = 20
+}
+
+output "workhorse" {
+  value = module.workhorse
+}

data/template/subspace/terraformrc.erb

@@ -0,0 +1,9 @@
+provider_installation {
+  filesystem_mirror {
+    path    = "<%= File.join(gem_path, 'terraform', 'modules') %>"
+    include = ["subspace/*/*"]
+  }
+  direct {
+    exclude = ["subspace/*/*"]
+  }
+}

data/terraform/modules/s3_backend/README

@@ -0,0 +1,2 @@
+This terraform configuration is used solely to provision the tfstate backend.  
+It makes an s3 bucket and a user and you should not need to really mess with this after the project is bootstrapped. 

data/terraform/modules/s3_backend/dynamodb.tf

@@ -0,0 +1 @@
+# Maybe later

data/terraform/modules/s3_backend/iam_user.tf

@@ -0,0 +1,38 @@
+resource "aws_iam_user" "ss" {
+  name = "subspace"
+  path = "/"
+
+  tags = {
+    Name = "Subspace IAM user"
+    Environment = "Global"
+  }
+}
+
+resource "aws_iam_access_key" "ss" {
+  user = aws_iam_user.ss.name
+
+  pgp_key = data.local_file.pgp_key.content_base64
+}
+
+resource "aws_iam_user_policy" "ss_s3" {
+  name = "ss_s3_user_policy"
+  user = aws_iam_user.ss.name
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:ListBucket",
+      "Resource": "arn:aws:s3:::${local.state_bucket_name}"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:PutObject"],
+      "Resource": "arn:aws:s3:::${local.state_bucket_name}/*"
+    }
+  ]
+}
+EOF
+}

data/terraform/modules/s3_backend/main.tf

@@ -0,0 +1,39 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+}
+
+# Variables
+variable aws_region {
+  type = string
+}
+
+variable project_name {
+  type = string
+}
+
+locals {
+  state_bucket_name = "subspace-backend-${var.project_name}"
+}
+
+provider "aws" {
+  region = var.aws_region
+}
+
+data "local_file" "pgp_key" {
+  filename = "../public-key-binary.gpg"
+}
+
+# Outputs
+
+output "subspace_aws_access_key_id" {
+  value = aws_iam_access_key.ss.id
+}
+
+output "subspace_aws_encrypted_secret_access_key" {
+  value = aws_iam_access_key.ss.encrypted_secret
+}

data/terraform/modules/s3_backend/state_bucket.tf

@@ -0,0 +1,14 @@
+# S3 Bucket for storing state
+resource "aws_s3_bucket" "state_bucket" {
+  bucket = local.state_bucket_name
+  acl    = "private"
+
+  versioning {
+    enabled = true
+  }
+
+  tags = {
+    Name        = "Subspace Terraform State"
+    Environment = "Global"
+  }
+}