subspace 2.5.4 → 2.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80d8c1f6e9cebb238b6afa922f2175d9d7c38e64ff64020b83b247b297d0a2bf
-  data.tar.gz: '07843c2f8262ce9d067318aeb6c6505fc8c96aafe7cd592b61ff2b381ca15c2e'
+  metadata.gz: 82942611925eb85847f321e3fc375f02fcd3c49fdde254012aa276792c309437
+  data.tar.gz: 58a200b894ab4a697c57a8db271ad36db1ba3cf304ee9045f903e872ec52cb14
 SHA512:
-  metadata.gz: '079d75a0072666cd49beb2ca889aedc1b7f8d54d41d29eac51685e7e95abf76af45c249f8ac474367e898aa528c4ab215de46ca95a91ce178dd5c9fa6a42c23d'
-  data.tar.gz: 9883316a47394683e198015ed045929b1528e2d9c262ae30ef27e34c625dc8e006b089b7e88f7bd7c6ba68405d681b38aeda55b99db9a278982680b8f2604a35
+  metadata.gz: 4c8e6dc1a37b42a77c44bcb3ce8a9a794b662505a06aa838af810151e25e283d673358e8062fbc6da2e342aa5d00de336d8d38633e4a2e4027f0a81800ccc35e
+  data.tar.gz: 24a4eccfcb22e0e9a70b3fe9030ec3303add72ef1a97a4bec9d3c2ce0e30c881f37bee9c80f0b3d691be8649574c445035fdc5666ea6686eab345b8b22b93885

data/.ruby-version

@@ -1 +1 @@
-2.6.3
+2.7.4

data/CHANGELOG.md

@@ -2,7 +2,7 @@
 
 This is a [changelog](https://keepachangelog.com/en/0.3.0/).
 
-This project attempts to follow [semantic versioning](https://semver.org/)
+This project attempts to follow [semantic versioning](https://semver.org/).
 
 ## Known Bugs
 
@@ -10,6 +10,25 @@ This project attempts to follow [semantic versioning](https://semver.org/)
   * Not working on OSX - macs don't read from /etc/profile.d/
   * Stops showing color if you `sudo su`
 
+## Unreleased
+
+## 2.5.8
+  * Add a new role for configuring a monit-based resque server
+  * Auto-detect mitogen for speed
+
+## 2.5.7
+  * Add ability to set the timezone for servers instead of forcing to Central Time
+  * Update puma configuration to support puma 5 with puma-daemon
+  * Update letsencrypt to add certbot-nginx support for newer ubuntu
+
+## 2.5.6
+  * Fix sending security stats
+  * Make sure apt package acl is installed in common role so ansible can become a non-privileged user
+
+## 2.5.5
+  * Remove duplicate nginx role from playbook templates
+  * Don't send stats if there have been no upgrades
+
 ## 2.5.4
   * certbox => certbot
 

data/README.md

@@ -283,11 +283,11 @@ Installs logrotate and lets you configure logs for automatic rotation.  Example
 ## newrelic
 
 ## newrelic-infra
-This role will install the next-gen "Newrelic One" infrastructure agent which can perform a few different functions for newrelic.  The previous "newrelic" role is deprecated. 
+This role will install the next-gen "Newrelic One" infrastructure agent which can perform a few different functions for newrelic.  The previous "newrelic" role is deprecated.
 
-Variables: 
-    # Required, the newrelic license key you get after signing up. 
-    newrelic_license: "longhashthingyougetfromnewrelichere" 
+Variables:
+    # Required, the newrelic license key you get after signing up.
+    newrelic_license: "longhashthingyougetfromnewrelichere"
     # Optional - send logs to newrelic one's log aggregator.
     newrelic_logs:
       - name: rails-production
@@ -373,6 +373,14 @@ Installs redis on the server.
     # Change to * if you want tthis available everywhere.
     redis_bind: 127.0.0.1
 
+## resque
+
+Install monitoring and automatic startup for resque workers via monit. You MUST set the `job_queues` variable as follows:
+
+    job_queues:
+      - default
+      - mailers
+      - exports
 ## ruby-common
 
 Installs ruby on the machine.  YOu can set a version by picking off the download url and sha hash from ruby-lang.org
@@ -387,10 +395,14 @@ Installs ruby on the machine.  YOu can set a version by picking off the download
 
 This will install a monit script that keeps sidekiq running.  We spawn one sidekiq instance that manages as many queues as you need.  Varaibles of note:
 
+    # Process these background job queues
     job_queues:
       - default
       - mailers
 
+    # Number of sidekiq *processes* to run
+    sidekiq_concurrency: 1
+
 * Note that as of v0.4.13, we now also add a unique job queue for each host with its hostname.  This is handy if you need to assign a job to a specific host.  In general you should use named queues, but occasionally this is useful and there's no harm in having it there unused.
 
 Sidekiq uses redis by default, and rails connects to a redis running on localhost by default.  However, this role does not depend on redis since in production it's likely redis will be running elsewhere.  If you're provisioning a standalone server, make sure to include the redis role.
@@ -407,7 +419,14 @@ Thanks to the following repositories for making their roles available:
 * https://github.com/mtpereira/ansible-passenger
 
 
-# Development
+# Mitogen
+
+In order to dramatically speed up ansible, you can install Mitogen: https://github.com/mitogen-hq/mitogen/blob/master/docs/ansible_detailed.rst
+
+    pip install -g mitogen
+
+Subspace will automatically detect this and update your ansible.cfg file so it is blazing fast.
+
 
 ## Directory Structure
 
@@ -428,8 +447,8 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version:
 
   1. update the version number in `version.rb`
-  2. update the version number in motds
-  3. run `bundle exec rake release`
+  2. `gem build subspace.gemspec`
+  3. `gem push subspace-x.y.z.gem`
 
 This will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/ansible/roles/common/defaults/main.yml

@@ -2,3 +2,4 @@
   swap_space: 512M
   deploy_user: deploy
   send_stats: false
+  timezone: America/Chicago

data/ansible/roles/common/files/sudoers-service

@@ -1 +1 @@
-deploy ALL=(root) NOPASSWD: /usr/sbin/service
+deploy ALL=(root) NOPASSWD: /usr/bin/systemctl, /usr/sbin/service

data/ansible/roles/common/tasks/main.yml

@@ -125,6 +125,12 @@
       - maintenance
       - upgrade
 
+  - name: Install acl so ansible can become a non-privileged user
+    apt:
+      pkg: acl
+      state: present
+    become: true
+
   - name: Get os_upgrades stats
     shell:
       cmd: |
@@ -153,7 +159,7 @@
           key: os_upgrades
           value: "{{stats_os_upgrades.stdout}}"
           hostname: "{{hostname}}"
-    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    when: (send_stats == true) and (stats_url is defined) and (stats_api_key is defined) and (stats_os_upgrades.stdout | length > 0)
     tags:
       - maintenance
       - stats
@@ -197,9 +203,8 @@
   - name: Get os_security_upgrades stats
     shell:
       cmd: |
-        sed -n "/$(date '+%Y-%m')/,+2p" updates.log | # Groups of lines from the current month
+        grep -A 1 $(date +%Y-%m) updates.log | # Groups of lines from the current month
         grep 'security' | # Only lines matching 'security'
-        grep -P -o '(^\d+)' | #Extract the numbers at the beginning of the lines
         awk '{s+=$1} END {print s}' # Sum all the lines
     args:
       chdir: /opt/subspace
@@ -222,7 +227,7 @@
           key: os_security_upgrades
           value: "{{stats_os_security_upgrades.stdout}}"
           hostname: "{{hostname}}"
-    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    when: (send_stats == true) and (stats_url is defined) and (stats_api_key is defined) and (stats_os_security_upgrades.stdout | length > 0)
     tags:
       - maintenance
       - stats
@@ -236,9 +241,9 @@
       - maintenance
       - stats
 
-  - name: set timezone to America/Chicago
+  - name: set timezone
     timezone:
-      name: America/Chicago
+      name: "{{timezone}}"
     tags:
       - maintenance
 

data/ansible/roles/letsencrypt/tasks/modern.yml

@@ -6,6 +6,13 @@
       state: present
     with_items:
       - ca-certificates
+    
+  - name: Install certbot-nginx
+    become: true
+    when: "'nginx' in role_names"
+    apt:
+      pkg: python3-certbot-nginx
+      state: present
 
   - name: "Set certbot binary"
     set_fact:

data/ansible/roles/puma/templates/puma.rb

@@ -1,3 +1,11 @@
+begin
+  # Needed for Puma 5 + puma-damon, but built in to Puma 4
+  # https://github.com/kigster/puma-daemon
+  require 'puma/daemon'
+rescue LoadError => e
+  # Puma 4 has `daemonize` built in
+end
+
 # Change to match your CPU core count
 workers {{puma_workers}}
 # Min and Max threads per worker

data/ansible/roles/resque/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+  - name: Install resque monit script
+    template:
+      src: resque-monit-rc
+      dest: /etc/monit/conf-available/resque_{{project_name}}_{{rails_env}}
+    become: true
+
+  - name: Enable resque monit script
+    file:
+      src: /etc/monit/conf-available/resque_{{project_name}}_{{rails_env}}
+      dest: /etc/monit/conf-enabled/resque_{{project_name}}_{{rails_env}}
+      state: link
+    notify:
+      - reload_monit
+      - restart_monit

data/ansible/roles/resque/templates/resque-monit-rc

@@ -0,0 +1,4 @@
+check process resque
+  with pidfile /u/apps/{{project_name}}/shared/tmp/pids/resque.pid
+  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && RAILS_ENV={{rails_env}} QUEUES={{hostname}},{{ job_queues | join(',') }} BACKGROUND=yes PIDFILE=/u/apps/{{project_name}}/shared/tmp/pids/resque.pid bundle exec rake resque:work'" with timeout 30 seconds
+  stop program = "/bin/su - deploy -c 'kill -s TERM `cat /u/apps/{{project_name}}/shared/tmp/pids/resque.pid`'" with timeout 30 seconds

data/ansible/roles/resque/templates/resque-systemd.service

@@ -0,0 +1,47 @@
+[Unit]
+Description=resque
+# consider adding redis-server.service if Redis is local and systemd-managed.
+After=syslog.target network.target
+
+# See these pages for lots of options:
+#
+#   https://www.freedesktop.org/software/systemd/man/systemd.service.html
+#   https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+#
+# THOSE PAGES ARE CRITICAL FOR ANY LINUX DEVOPS WORK; read them multiple
+# times! systemd is a critical tool for all developers to know and understand.
+#
+[Service]
+#
+#      !!!!  !!!!  !!!!
+#
+Type=simple
+
+WorkingDirectory=/u/apps/{{project_name}}/current
+
+ExecStart="RAILS_ENV={{rails_env}} COUNT={{resque_concurrency}} QUEUES={{hostname}},{{ job_queues | join(',') }} BACKGROUND=yes PIDFILE=/u/apps/{{project_name}}/shared/tmp/pids/resque.pid bundle exec rake resque:work"
+
+# Uncomment this if you are going to use this as a system service
+# if using as a user service then leave commented out, or you will get an error trying to start the service
+# !!! Change this to your deploy user account if you are using this as a system service !!!
+User=deploy
+Group=deploy
+UMask=0002
+
+# Greatly reduce Ruby memory fragmentation and heap usage
+# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
+Environment=MALLOC_ARENA_MAX=2
+
+# if we crash, restart
+RestartSec=1
+Restart=on-failure
+
+# output goes to /var/log/syslog (Ubuntu) or /var/log/messages (CentOS)
+StandardOutput=syslog
+StandardError=syslog
+
+# This will default to "bundler" if we don't specify it
+SyslogIdentifier=resque
+
+[Install]
+WantedBy=multi-user.target

data/ansible/roles/sidekiq/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-  sidekiq_concurrency: 10
+  sidekiq_concurrency: 1

data/lib/subspace/commands/ansible.rb

@@ -13,6 +13,12 @@ module Subspace
       private
 
       def update_ansible_cfg
+        if `pip show mitogen` =~ /^Location: (.*?)$/m
+          @mitogen_path = $1
+          puts "🏎🚀🚅Mitogen found at #{@mitogen_path}.  WARP 9!....ENGAGE!🚀"
+        else
+          puts "Mitogen not detected.  Ansible will be slow.  Run `pip install mitogen` to fix."
+        end
         template! "ansible.cfg"
       end
     end

data/lib/subspace/version.rb

@@ -1,3 +1,3 @@
 module Subspace
-  VERSION = "2.5.4"
+  VERSION = "2.5.8"
 end

data/template/provision/ansible.cfg.erb

@@ -3,7 +3,14 @@ inventory = hosts
 forks     = 10
 roles_path = ./roles:<%= File.join(gem_path, 'ansible', 'roles') %>:/etc/ansible/roles
 vault_password_file = .vault_pass
+# Uncomment to add timestamps to tasks to find slow ones.
+# callback_whitelist = profile_tasks
+
+<% if @mitogen_path %>
+strategy_plugins = <%= @mitogen_path %>/ansible_mitogen/plugins/strategy
+strategy = mitogen_linear
+<% end %>
 
 [ssh_connection]
-pipelining=True
-control_path = %(directory)s/%%h-%%p-%%r
+pipelining = True
+control_path = /tmp/subspace-control-%%h-%%p-%%r

data/template/provision/playbook.yml.erb

@@ -12,9 +12,8 @@
       - ruby-common
       - rails
       - puma
-      - nginx
       - letsencrypt
-      - nginx # This is included twice intentionally. I think there is a bug that is fixed by running it both before and after the letsencrypt role.
+      - nginx
       - postgresql
       - monit
       - logrotate

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: subspace
 version: !ruby/object:Gem::Version
-  version: 2.5.4
+  version: 2.5.8
 platform: ruby
 authors:
 - Brian Samson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-02 00:00:00.000000000 Z
+date: 2022-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -225,6 +225,9 @@ files:
 - ansible/roles/rails/templates/database.yml
 - ansible/roles/redis/defaults/main.yml
 - ansible/roles/redis/tasks/main.yml
+- ansible/roles/resque/tasks/main.yml
+- ansible/roles/resque/templates/resque-monit-rc
+- ansible/roles/resque/templates/resque-systemd.service
 - ansible/roles/ruby-common/README.md
 - ansible/roles/ruby-common/defaults/main.yml
 - ansible/roles/ruby-common/meta/main.yml
@@ -310,7 +313,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.4
 signing_key:
 specification_version: 4
 summary: Ansible-based server provisioning for rails projects