subspace 2.5.10 → 3.0.0

data/ansible/roles/memcache/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+  memcache_bind: 127.0.0.1

data/ansible/roles/memcache/tasks/main.yml

@@ -3,4 +3,19 @@
   apt: update_cache=yes cache_valid_time=86400
 
 - name: Install Memcached.
-  apt: name=memcached state=present
+  apt:
+    name: memcached
+    state: present
+
+- name: Configure memcache bind address
+  lineinfile:
+    path: /etc/memcached.conf
+    regex: "^(#\\s*)?-l"
+    state: present
+    line: "-l {{memcache_bind}}"
+
+- name: restart memcached
+  systemd:
+    name: memcached
+    state: restarted
+    enabled: yes

data/ansible/roles/newrelic-infra/tasks/main.yml

@@ -1,12 +1,12 @@
 ---
   - name: Add New Relic apt key
     apt_key:
-      url: https://download.newrelic.com/infrastructure_agent/gpg/newrelic-infra.gpg 
+      url: https://download.newrelic.com/infrastructure_agent/gpg/newrelic-infra.gpg
       state: present
     become: true
 
   - name: create license key
-    copy: 
+    copy:
       dest: "/etc/newrelic-infra.yml"
       content: |
         license_key: {{newrelic_license}}
@@ -27,7 +27,7 @@
   - name: Configure application log forwarding if enabled
     when: "{{ newrelic_logs|length }}"
     become: true
-    template: 
+    template:
       dest: "/etc/newrelic-infra/logging.d/subspace.yml"
       src: logs.yml.j2
     notify: Restart newrelic-infra

data/ansible/roles/nginx/tasks/main.yml

@@ -25,10 +25,19 @@
     state: link
   become: true
 
-- name: Restart nginx
-  action: service name=nginx state=restarted
+- name: Restart nginx (systemd)
   become: true
+  systemd:
+    state: restarted
+    daemon_reload: yes
+    name: nginx
+
+- name: Enable nginx to start on reboot
+  become: true
+  systemd:
+    name: nginx
+    enabled: true
 
 - name: Nginx is installed
   set_fact:
-    nginx_installed: true
+    nginx_installed: true

data/ansible/roles/puma/tasks/main.yml

@@ -6,31 +6,43 @@
   tags: puma
 
 - name: Add puma shared/config
-  template: src=puma.rb dest=/u/apps/{{project_name}}/shared/config/puma/{{rails_env}}.rb group=deploy owner=deploy force=yes mode=755
+  template:
+    src: puma.rb
+    dest: /u/apps/{{project_name}}/shared/config/puma/{{rails_env}}.rb
+    group: deploy
+    owner: deploy
+    force: yes
+    mode: 0755
   tags: puma
 
 - name: Make shared/tmp/sockets
   file: path=/u/apps/{{project_name}}/shared/tmp/sockets group=deploy owner=deploy state=directory
   tags: tmp
 
-- name: Install puma monit script
+- name: Install systemd script
+  become: true
+  template:
+    src: puma-systemd.service
+    dest: /etc/systemd/system/puma.service
+
+- name: Install systemd socket
+  become: true
   template:
-    src: puma-monit-rc
-    dest: /etc/monit/conf-available/puma_{{project_name}}_{{rails_env}}
-
-- name: Clean up old puma monit scripts
-  shell: rm -f /etc/monit/conf.d/puma_*
-
-- name: mkdir /etc/monit/conf-enabled
-  file:
-    path: /etc/monit/conf-enabled
-    state: directory
-
-- name: Enable puma monit script
-  file:
-    src: /etc/monit/conf-available/puma_{{project_name}}_{{rails_env}}
-    dest: /etc/monit/conf-enabled/puma_{{project_name}}_{{rails_env}}
-    state: link
-  notify:
-    - restart_monit
+    src: puma-systemd.socket
+    dest: /etc/systemd/system/puma.socket
+
+- name: enable systemd service
+  become: true
+  systemd:
+    name: puma.service
+    enabled: yes
+    daemon_reload: yes
+
+- name: disable systemd socket
+  become: true
+  systemd:
+    name: puma.socket
+    enabled: no
+    daemon_reload: yes
+
 

data/ansible/roles/puma/templates/puma-systemd.service

@@ -0,0 +1,37 @@
+[Unit]
+Description=Puma HTTP Server
+After=network.target
+
+# Uncomment for socket activation (see below)
+# Requires=puma.socket
+
+[Service]
+# Puma supports systemd's `Type=notify` and watchdog service
+# monitoring, if the [sd_notify](https://github.com/agis/ruby-sdnotify) gem is installed,
+# as of Puma 5.1 or later.
+# On earlier versions of Puma or JRuby, change this to `Type=simple` and remove
+# the `WatchdogSec` line.
+Type=simple
+
+# If your Puma process locks up, systemd's watchdog will restart it within seconds.
+# WatchdogSec=10
+
+# Preferably configure a non-privileged user
+User=deploy
+
+WorkingDirectory=/u/apps/{{project_name}}/current
+
+# Helpful for debugging socket activation, etc.
+# Environment=PUMA_DEBUG=1
+Environment=RAILS_ENV={{rails_env}}
+
+# SystemD will not run puma even if it is in your path. You must specify
+# an absolute URL to puma. For example /usr/local/bin/puma
+
+# Variant: Use `bundle exec --keep-file-descriptors puma` instead of binstub
+ExecStart=/usr/local/bin/bundle exec --keep-file-descriptors puma -C /u/apps/{{project_name}}/current/config/puma/{{rails_env}}.rb
+
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

data/ansible/roles/puma/templates/puma-systemd.socket

@@ -0,0 +1,14 @@
+# /etc/systemd/system/puma.socket
+
+[Unit]
+Description=Puma HTTP Server Accept Sockets
+
+[Socket]
+ListenStream=0.0.0.0:9292
+
+NoDelay=true
+ReusePort=true
+Backlog=1024
+
+[Install]
+WantedBy=sockets.target

data/ansible/roles/puma/templates/puma.rb

@@ -1,10 +1,14 @@
+{% if monit_installed is defined %}
 begin
   # Needed for Puma 5 + puma-damon, but built in to Puma 4
   # https://github.com/kigster/puma-daemon
+  # however not needed if we're using systemd which is the future
   require 'puma/daemon'
 rescue LoadError => e
+  daemonize
   # Puma 4 has `daemonize` built in
 end
+{% endif %}
 
 # Change to match your CPU core count
 workers {{puma_workers}}
@@ -23,8 +27,6 @@ bind "tcp://127.0.0.1:9292"
 # Logging
 stdout_redirect "#{app_dir}/log/puma.stdout.log", "#{app_dir}/log/puma.stderr.log", true
 
-# Set master PID and state locations
-daemonize
 pidfile "/u/apps/{{project_name}}/shared/tmp/pids/puma.pid"
 state_path "/u/apps/{{project_name}}/shared/tmp/pids/puma.state"
 activate_control_app

data/ansible/roles/rails/defaults/main.yml

@@ -1,9 +1,2 @@
 ---
-database_pool: 5
-database_name: "{{project_name}}_{{rails_env}}"
-database_user: "{{project_name}}"
-database_adapter: postgresql
-job_queues:
-  - default
-  - mailers
 send_stats: false

data/ansible/roles/redis/tasks/main.yml

@@ -1,10 +1,29 @@
 ---
-  - name: Install Redis
+  - name: Add an Apt signing key for redis repo
+    ansible.builtin.apt_key:
+      url: https://packages.redis.io/gpg
+      state: present
+
+  - name: Add redis repository into sources list
+    ansible.builtin.apt_repository:
+      repo: deb https://packages.redis.io/deb {{ ansible_distribution_release }} main
+      state: present
+    register: redis_apt_repo
+
+  - name: Purge distro redis package
+    apt:
+      name: redis-server
+      state: absent
+      purge: true
+    when: redis_apt_repo.changed
+
+  - name: Install Redis from official repo
     become: true
     apt:
-      pkg: redis-server
-      state: present
+      name: redis-server
+      state: latest
       update_cache: true
+
   - name: Set bind IP
     become: true
     lineinfile:
@@ -12,3 +31,9 @@
       regexp: '^bind '
       line: 'bind {{redis_bind}}'
       state: present
+
+  - name: restart redis
+    become: true
+    systemd:
+      name: redis
+      state: restarted

data/ansible/roles/resque/tasks/main.yml

@@ -1,15 +1,14 @@
 ---
-  - name: Install resque monit script
-    template:
-      src: resque-monit-rc
-      dest: /etc/monit/conf-available/resque_{{project_name}}_{{rails_env}}
+  - name: Install systemd resque script
     become: true
+    template:
+      src: resque-systemd.service
+      dest: /etc/systemd/system/resque.service
 
-  - name: Enable resque monit script
-    file:
-      src: /etc/monit/conf-available/resque_{{project_name}}_{{rails_env}}
-      dest: /etc/monit/conf-enabled/resque_{{project_name}}_{{rails_env}}
-      state: link
-    notify:
-      - reload_monit
-      - restart_monit
+  - name: Enable systemd resque service
+    become: true
+    systemd:
+      name: resque
+      daemon_reload: true
+      enabled: yes
+      state: started

data/ansible/roles/resque/templates/resque-systemd.service

@@ -15,11 +15,13 @@ After=syslog.target network.target
 #
 #      !!!!  !!!!  !!!!
 #
-Type=simple
+Type=forking
 
 WorkingDirectory=/u/apps/{{project_name}}/current
 
-ExecStart="RAILS_ENV={{rails_env}} COUNT={{resque_concurrency}} QUEUES={{hostname}},{{ job_queues | join(',') }} BACKGROUND=yes PIDFILE=/u/apps/{{project_name}}/shared/tmp/pids/resque.pid bundle exec rake resque:work"
+ExecStart=/usr/local/bin/bundle exec rake resque:work
+ExecStop=/bin/kill -s QUIT $MAINPID
+PIDFile=/u/apps/{{project_name}}/shared/tmp/pids/resque.pid
 
 # Uncomment this if you are going to use this as a system service
 # if using as a user service then leave commented out, or you will get an error trying to start the service
@@ -31,6 +33,11 @@ UMask=0002
 # Greatly reduce Ruby memory fragmentation and heap usage
 # https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
 Environment=MALLOC_ARENA_MAX=2
+Environment=RAILS_ENV={{rails_env}}
+Environment=COUNT=1
+Environment=QUEUES={{ job_queues | join(',') }}
+Environment=BACKGROUND=yes
+Environment=PIDFILE=/u/apps/{{project_name}}/shared/tmp/pids/resque.pid
 
 # if we crash, restart
 RestartSec=1
@@ -44,4 +51,4 @@ StandardError=syslog
 SyslogIdentifier=resque
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

data/ansible/roles/ruby-common/tasks/main.yml

@@ -71,22 +71,7 @@
   command: "{{ ruby_location }}/bin/gem update --system"
   become: true
 
-- name: Remove old bundler bin
-  file:
-    path: "{{ ruby_location }}/bin/bundle"
-    state: absent
-  become: true
-
-- name: Uninstall Bundler
-  gem:
-    name: bundler
-    state: absent
-    user_install: no
-    executable: "{{ ruby_location }}/bin/gem"
-  become: true
-  ignore_errors: yes
-
-- name: Install Bundler
+- name: Install/update Bundler
   shell: "{{ ruby_location }}/bin/gem install bundler -v {{ bundler_version }}"
   become: true
 

data/ansible/roles/sidekiq/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-  sidekiq_concurrency: 1
+  sidekiq_workers: 1

data/ansible/roles/sidekiq/tasks/main.yml

@@ -1,19 +1,15 @@
 ---
-  - name: Install sidekiq monit script
-    template:
-      src: sidekiq-monit-rc
-      dest: /etc/monit/conf-available/sidekiq_{{project_name}}_{{rails_env}}
+  - name: Install systemd sidekiq script
     become: true
+    template:
+      src: sidekiq-systemd.service
+      dest: /etc/systemd/system/sidekiq.service
 
-  - name: Clean up old sidekiq monit scripts
-    shell: rm -f /etc/monit/conf.d/sidekiq_*
-
-  - name: Enable sidekiq monit script
-    file:
-      src: /etc/monit/conf-available/sidekiq_{{project_name}}_{{rails_env}}
-      dest: /etc/monit/conf-enabled/sidekiq_{{project_name}}_{{rails_env}}
-      state: link
-    notify:
-      - reload_monit
-      - restart_monit
+  - name: Enable systemd sidekiq service
+    become: true
+    systemd:
+      name: sidekiq
+      enabled: yes
+      daemon_reload: true
 
+# TODO Read the gemfile and make sure they have sidekiq 6????

data/ansible/roles/sidekiq/templates/sidekiq-monit-rc

@@ -1,4 +1,4 @@
 check process sidekiq
   with pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid
-  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} -c {{sidekiq_concurrency}} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
+  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} -c {{sidekiq_workers}} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
   stop program = "/bin/su - deploy -c 'kill -s TERM `cat /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid`'" with timeout 30 seconds

data/ansible/roles/sidekiq/templates/sidekiq-systemd.service

@@ -0,0 +1,63 @@
+#
+# This file tells systemd how to run Sidekiq as a 24/7 long-running daemon.
+#
+# Use `journalctl -u sidekiq -rn 100` to view the last 100 lines of log output.
+#
+[Unit]
+Description=sidekiq
+# start us only once the network and logging subsystems are available,
+# consider adding redis-server.service if Redis is local and systemd-managed.
+After=syslog.target network.target
+
+# See these pages for lots of options:
+#
+#   https://www.freedesktop.org/software/systemd/man/systemd.service.html
+#   https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+#
+# THOSE PAGES ARE CRITICAL FOR ANY LINUX DEVOPS WORK; read them multiple
+# times! systemd is a critical tool for all developers to know and understand.
+#
+[Service]
+#
+#      !!!!  !!!!  !!!!
+#
+# As of v6.0.6, Sidekiq automatically supports systemd's `Type=notify` and watchdog service
+# monitoring. If you are using an earlier version of Sidekiq, change this to `Type=simple`
+# and remove the `WatchdogSec` line.
+#
+#      !!!!  !!!!  !!!!
+#
+Type=notify
+# If your Sidekiq process locks up, systemd's watchdog will restart it within seconds.
+WatchdogSec=10
+
+WorkingDirectory=/u/apps/{{project_name}}/current
+ExecStart=/usr/local/bin/bundle exec sidekiq -e {{rails_env}} --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} -c {{sidekiq_workers}}
+
+# Use `systemctl kill -s TSTP sidekiq` to quiet the Sidekiq process
+
+# Uncomment this if you are going to use this as a system service
+# if using as a user service then leave commented out, or you will get an error trying to start the service
+# !!! Change this to your deploy user account if you are using this as a system service !!!
+User=deploy
+Group=deploy
+UMask=0002
+
+# Greatly reduce Ruby memory fragmentation and heap usage
+# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
+Environment=MALLOC_ARENA_MAX=2
+Environment=RAILS_ENV={{rails_env}}
+
+# if we crash, restart
+RestartSec=1
+Restart=on-failure
+
+# output goes to /var/log/syslog (Ubuntu) or /var/log/messages (CentOS)
+StandardOutput=syslog
+StandardError=syslog
+
+# This will default to "bundler" if we don't specify it
+SyslogIdentifier=sidekiq
+
+[Install]
+WantedBy=multi-user.target

data/ansible/roles/tailscale/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+  tailscale_options: ""

data/ansible/roles/tailscale/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+  - name: "Add Tailscale apt key"
+    become: true
+    apt_key:
+      url: https://pkgs.tailscale.com/stable/ubuntu/{{ansible_distribution_release}}.gpg
+      state: present
+
+  - name: "Add Tailscale apt repos"
+    become: true
+    apt_repository:
+      repo: "deb https://pkgs.tailscale.com/stable/ubuntu {{ansible_distribution_release}} main"
+      state: present
+
+  - name: "Install tailscale from api"
+    apt:
+      name: tailscale
+      state: latest
+      update_cache: yes
+
+  - name: "Join the tailnet"
+    become: true
+    command: tailscale up --ssh --auth-key={{tailscale_auth_key}} --hostname={{project_name}}-{{hostname}} --accept-risk=lose-ssh {{tailscale_options}}

data/bin/console

@@ -6,9 +6,5 @@ require "subspace"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
 require "irb"
 IRB.start

data/exe/subspace

@@ -1,5 +1,4 @@
-#!/usr/bin/env ruby
-
+#!/usr/bin/env ruby_executable_hooks
 
 require 'subspace/cli'
 Subspace::Cli.new.run

data/lib/subspace/cli.rb

@@ -7,11 +7,14 @@ require 'subspace'
 require 'subspace/commands/base'
 require 'subspace/commands/bootstrap'
 require 'subspace/commands/configure'
+require 'subspace/commands/exec'
 require 'subspace/commands/init'
+require 'subspace/commands/inventory'
 require 'subspace/commands/override'
 require 'subspace/commands/provision'
 require 'subspace/commands/ssh'
-require 'subspace/commands/vars'
+require 'subspace/commands/secrets'
+require 'subspace/commands/terraform'
 require 'subspace/commands/maintain'
 require 'subspace/commands/maintenance_mode.rb'
 
@@ -30,11 +33,15 @@ class Subspace::Cli
     end
 
     command :init do |c|
-      c.syntax = 'subspace init [vars]'
+      c.syntax = 'subspace init'
       c.summary = 'Run without options to initialize subspace.'
       c.description = 'Some initialization routines can be run indiviaully, useful for upgrading'
-      c.example 'init a new project', 'subspace init'
-      c.example 'create the new style application.yml vars template', 'subspace init vars'
+      c.example 'init a new project with one default environment (default staging)', 'subspace init'
+      c.example 'create a new fully automated production environment configuration', 'subspace init --terraform --env production'
+      c.option '--ansible', 'initialize ansible for managing individual servers'
+      c.option '--terraform', 'Initialize terraform for managing infrastructure'
+      c.option '--env STRING', 'Initialize configuration for a new environment'
+      c.option '--template [staging|production]', 'Use non-default template for this environment (default=staging)'
       c.when_called Subspace::Commands::Init
     end
 
@@ -47,9 +54,6 @@ class Subspace::Cli
       c.option '--password', "Ask for a password instead of using ssh keys"
       c.option '--yum', "Use yum instead of apt to install python"
       c.option "-i", "--private-key PRIVATE-KEY", "Alias for private-key"
-      Subspace::Commands::Bootstrap::PASS_THROUGH_PARAMS.each do |param_name|
-        c.option "--#{param_name} #{param_name.upcase}", "Passed directly through to ansible-playbook command"
-      end
       c.when_called Subspace::Commands::Bootstrap
     end
 
@@ -64,17 +68,34 @@ class Subspace::Cli
       c.when_called Subspace::Commands::Provision
     end
 
+    command :tf do |c|
+      c.syntax = 'subspace tf [environment]'
+      c.summary = "Execute a terraform plan with the option to apply the plan after review"
+      c.when_called Subspace::Commands::Terraform
+    end
+
     command :ssh do |c|
       c.syntax = 'subspace ssh [options]'
       c.summary = 'ssh to the remote server as the administrative user'
       c.description = ''
-      c.option '--user USER', "Use a different user (eg deploy).  Default is the ansible_ssh_user"
+      c.option '--user USER', "Use a different user (eg deploy).  Default is the ansible_user"
       Subspace::Commands::Ssh::PASS_THROUGH_PARAMS.each do |param_name|
         c.option "-#{param_name} #{param_name.upcase}", "Passed directly through to ssh command"
       end
       c.when_called Subspace::Commands::Ssh
     end
 
+    command :exec do |c|
+      c.syntax = 'subspace exec <host-spec> "<statement>" [options]'
+      c.summary = 'execute <statement> on all hosts matching <host-spec>'
+      c.description = ''
+      c.option '--user USER', "Use a different user (eg deploy).  Default is the ansible_user"
+      Subspace::Commands::Exec::PASS_THROUGH_PARAMS.each do |param_name|
+        c.option "-#{param_name} #{param_name.upcase}", "Passed directly through to ssh command"
+      end
+      c.when_called Subspace::Commands::Exec
+    end
+
     command :configure do |c, args|
       c.syntax = 'subspace configure'
       c.summary = "Regenerate all of the ansible configuration files. You don't normally need to run this."
@@ -89,15 +110,17 @@ class Subspace::Cli
       c.when_called Subspace::Commands::Override
     end
 
-    command :vars do |c, args|
-      c.syntax = 'subspace vars [environment]'
+    command :secrets do |c, args|
+      c.syntax = 'subspace secrets [environment]'
       c.summary = 'View or edit the encrypted variables for an environment'
-      c.description = """By default, this will simply show the variables for a specific environemnt.
-                         You can also edit variables, and we expect the functionality here to grow in the future.
-                         Running `subspace vars development --create` is usually a great way to bootstrap a new development environment."""
+      c.description = <<~EOS
+        By default, this will simply show the variables for a specific environemnt.
+        You can also edit variables, and we expect the functionality here to grow in the future.
+        Running `subspace secrets development --create` is usually a great way to bootstrap a new development environment.
+        EOS
       c.option '--edit', "Edit the variables instead of view"
       c.option '--create', "Create config/application.yml with the variables from the specified environment"
-      c.when_called Subspace::Commands::Vars
+      c.when_called Subspace::Commands::Secrets
     end
 
     command :maintain do |c, args|
@@ -124,6 +147,20 @@ class Subspace::Cli
       c.when_called Subspace::Commands::MaintenanceMode
     end
 
+    command :inventory do |c, args|
+      c.syntax = 'subspace inventory <command>'
+      c.summary = 'Manage, manipulate, and other useful inventory-related functions'
+      c.description = <<~EOS
+        Available inventory commands:
+
+        capistrano - generate config/deploy/[env].rb.  Requires the --env option.
+        list       - list the current inventory as understood by subspace.
+        keyscan    - Update ~/.known_hosts with new host key fingerprints
+        EOS
+      c.option "--env ENVIRONMENT", "Optional: Limit function to a specific environment (aka group)"
+      c.when_called Subspace::Commands::Inventory
+    end
+
     run!
   end
 end