subspace 2.2.3 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a986a32d45258059489792b7da873b584d4b90ae67c94d3e3e96dc8075c5f6e8
-  data.tar.gz: b0169f7307a741609e6bba7c3cfd18ef0d6b8316d1c168eeea39c1bd2ac9234e
+  metadata.gz: f235d8197ea5831cdf3006f8f180ae826f524665a5965965b6292b6403adc206
+  data.tar.gz: 8c6d3e7a879a8fda045b8c30a94c0631fa73a21d8f3509b771a755dd947dd13a
 SHA512:
-  metadata.gz: 8103b832b31b022559e5a2b41117bb573fb564ab6895e451ee2f8d793d872057eaad680ad36a96cb56acb5471913e5c4048d2e5ce9868f912afa8d6de6cf518f
-  data.tar.gz: 19cafc562c26440dc90e60e2e83c49d168518c4ee151267625a03e403b28c2545e5c6c15fc176d3e2d64bf64b6510291eb741c1fb4260fe48fff377fdc67db3a
+  metadata.gz: 2de55ab4d546444eeaf8d98cdc6d42b8edb589c3ef500f56b7797f2cbff81d5feeceeb069b573d72d049f18048f6a72e3085226731810b0d8fd6de266d8981ce
+  data.tar.gz: 1d58d3a3df528cb398ae1c4feafd085cecad96ca56063bddd117002b43cbf485f32fe8bf8fb879bb68918095cbea00aa228717782bb6bdedc92badba094e21f4

data/CHANGELOG.md

@@ -10,6 +10,27 @@ This project attempts to follow [semantic versioning](https://semver.org/)
   * Not working on OSX - macs don't read from /etc/profile.d/
   * Stops showing color if you `sudo su`
 
+## 2.4
+  Lots of modifications for ubuntu 20.04, which has python3 as a default
+
+  * Change letsencrypt to pull from apt instead of build from source (backwards compatible)
+  * Change postgres to a cleaner install and deprecate the old zenoamaro role
+  * postgresql_version is now a required variable and no longer defaults to 9.4
+  * Better detection of web servers
+
+## 2.3.3
+  * Tweak the way that different roles are detected to be more reliable
+
+## 2.3.2
+  * Update papertrail to latest version of remote_syslog2 and add support for nginx logs
+
+## 2.3.1
+  * Sidekiq concurrency actually works
+
+## 2.3.0
+  * Grab linux kernel to send as stats
+  * Grab psql version to send as stats
+
 ## 2.2.3
   * Add PATH to crontab for letsencrypt auto renewal
   * log letsencrypt crontab to /var/log/cron.log

data/ansible/roles/apache/tasks/main.yml

@@ -52,3 +52,7 @@
       state: link
     become: true
     notify: apache restart
+
+  - name: Apache is installed
+    set_fact:
+      apache_installed: true

data/ansible/roles/collectd/tasks/main.yml

@@ -47,7 +47,7 @@
       dest: /etc/collectd/collectd.conf.d/apache2.conf
     become: true
     notify: restart collectd
-    when: "'apache' in role_names"
+    when: apache_installed is defined
 
   - name: create puma config
     template:
@@ -70,7 +70,7 @@
       dest: /etc/collectd/collectd.conf.d/nginx.conf
     become: true
     notify: restart collectd
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
 
   - name: create rails_lograge config
     template:

data/ansible/roles/common/tasks/main.yml

@@ -252,4 +252,30 @@
     tags:
       - maintenance
 
+  - name: Grab OS version
+    shell: uname --kernel-release
+    register: stats_os_version
+    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    tags:
+      - maintenance
+      - stats
+
+  - name: Send OS stats to URL
+    uri:
+      url: "{{stats_url}}"
+      method: POST
+      headers:
+        X-API-Version: 1
+        X-Client-Api-key: "{{stats_api_key}}"
+      body_format: json
+      body:
+        client_stat:
+          key: os_version
+          value: "{{stats_os_version.stdout}}"
+          hostname: "{{hostname}}"
+    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    tags:
+      - maintenance
+      - stats
+
   - import_tasks: swap.yml

data/ansible/roles/common/templates/motd

@@ -4,7 +4,7 @@ This server brought to you by:
 \___ \| | | | '_ \___ \| '_ \ / _` |/ __/ _ \
  ___) | |_| | |_) |__) | |_) | (_| | (_|  __/
 |____/ \__,_|_.__/____/| .__/ \__,_|\___\___|
-                       |_|             v2.2.3
+                       |_|             v2.4.0
 ~~~ https://github.com/tenforwardconsulting/subspace ~~~
 
 If you need to make configuration changes to the server, please modify the

data/ansible/roles/letsencrypt/tasks/legacy.yml

@@ -0,0 +1,44 @@
+---
+  - name: Install certbot dependencies
+    become: true
+    apt:
+      pkg: "{{item}}"
+      state: present
+    with_items:
+      - augeas-lenses
+      - ca-certificates
+      - dialog
+      - gcc
+      - libaugeas0
+      - libffi-dev
+      - libpython-dev
+      - libpython2.7-dev
+      - libssl-dev
+      - python
+      - python-dev
+      - python-setuptools
+      - python-virtualenv
+      - python2.7
+      - python2.7-dev
+
+  - name: "Create certbot dir"
+    become: true
+    file:
+      path: "{{certbot_dir}}"
+      state: directory
+      mode: 0755
+
+  - name: "Set certbot binary"
+    set_fact:
+      certbot_bin: "{{certbot_dir}}/certbot_auto"
+
+  - name: Get certbot
+    become: true
+    get_url:
+      url: "https://dl.eff.org/certbot-auto"
+      dest: "{{certbot_bin}}"
+      mode: a+x
+
+
+
+

data/ansible/roles/letsencrypt/tasks/main.yml

@@ -1,38 +1,32 @@
 ---
-  - name: Install certbot dependencies
+  - name: Ensure nginx is installed (first time)
     become: true
     apt:
-      pkg: "{{item}}"
+      pkg: nginx
       state: present
-    with_items:
-      - augeas-lenses
-      - ca-certificates
-      - dialog
-      - gcc
-      - libaugeas0
-      - libffi-dev
-      - libpython-dev
-      - libpython2.7-dev
-      - libssl-dev
-      - python
-      - python-dev
-      - python-setuptools
-      - python-virtualenv
-      - python2.7-dev
-
-  - name: "Create certbot dir"
-    become: true
-    file:
-      path: "{{certbot_dir}}"
-      state: directory
-      mode: 0755
+    when: "'nginx' in role_names"
 
-  - name: Get certbot
+  - name: Attempt to install certbot from APT
     become: true
-    get_url:
-      url: "https://dl.eff.org/certbot-auto"
-      dest: "{{certbot_dir}}/certbot-auto"
-      mode: a+x
+    ignore_errors: true
+    apt:
+      pkg: certbox
+      state: present
+
+  - name: "Detect if certbot was installed via APT"
+    shell: dpkg-query -W 'certbot'
+    ignore_errors: true
+    register: apt_certbot
+
+  - name: "Modern Letsencrypt Installation (py3, apt version)"
+    include_tasks: modern.yml
+    when: apt_certbot is succeeded
+
+  - name: "Legacy Letsencrypt Installation (py2, from source)"
+    include_tasks: legacy.yml
+    when: apt_certbot is failed
+
+# Post install configuration
 
   - name: shutdown webserver for standalone mode
     debug: msg="Shutdown webserver"
@@ -50,21 +44,21 @@
   - name: Run default
     when: le_ssl_certs is not defined
     become: true
-    command: "{{certbot_dir}}/certbot-auto certonly --email {{letsencrypt_email}} --domains {{([server_name] + server_aliases) | join(',')}} --standalone --agree-tos --expand --non-interactive"
+    command: "{{certbot_bin}} certonly --email {{letsencrypt_email}} --domains {{([server_name] + server_aliases) | join(',')}} --standalone --agree-tos --expand --non-interactive"
 
   - name: Generate SSL Certificates
     become: true
     with_items: "{{le_ssl_certs|default([])}}"
-    command: "{{certbot_dir}}/certbot-auto certonly --email {{letsencrypt_email}} --domains {{item.domains | join(',')}} --cert-name {{item.cert_name}} --standalone --agree-tos --expand --non-interactive"
+    command: "{{certbot_bin}} certonly --email {{letsencrypt_email}} --domains {{item.domains | join(',')}} --cert-name {{item.cert_name}} --standalone --agree-tos --expand --non-interactive"
 
   - name: Update nginx default options
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
     get_url:
       url: https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf
       dest: /etc/letsencrypt/options-ssl-nginx.conf
 
   - name: Update apache default options
-    when: "'apache' in role_names"
+    when: apache_installed is defined
     get_url:
       url: https://raw.githubusercontent.com/certbot/certbot/master/certbot-apache/certbot_apache/options-ssl-apache.conf
       dest: /etc/letsencrypt/options-ssl-apache.conf
@@ -82,20 +76,20 @@
 
   - name: Setup cron job to auto renew
     become: true
-    when: "'apache' in role_names"
+    when: apache_installed is defined
     cron:
       name: Auto-renew SSL
-      job: "{{certbot_dir}}/certbot-auto renew --no-self-upgrade --apache >> /var/log/cron.log 2>&1"
+      job: "{{certbot_bin}} renew --no-self-upgrade --apache >> /var/log/cron.log 2>&1"
       hour: "0"
       minute: "33"
       state: present
 
   - name: Setup cron job to auto renew
     become: true
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
     cron:
       name: Auto-renew SSL
-      job: "{{certbot_dir}}/certbot-auto renew --no-self-upgrade --nginx >> /var/log/cron.log 2>&1"
+      job: "{{certbot_bin}} renew --no-self-upgrade --nginx >> /var/log/cron.log 2>&1"
       hour: "0"
       minute: "33"
-      state: present
+      state: present

data/ansible/roles/letsencrypt/tasks/modern.yml

@@ -0,0 +1,13 @@
+---
+  - name: Install ca-certificates
+    become: true
+    apt:
+      pkg: "{{item}}"
+      state: present
+    with_items:
+      - ca-certificates
+
+  - name: "Set certbot binary"
+    set_fact:
+      certbot_bin: "certbot"
+

data/ansible/roles/nginx-rails/tasks/main.yml

@@ -25,3 +25,14 @@
     dest: /etc/nginx/sites-enabled/{{project_name}}-ssl
     state: "{{ (ssl_enabled and nginx_ssl_config is defined) | ternary('link', 'absent') }}"
   become: true
+
+- name: Enable a default server if one is not defined in the app
+  template:
+    src: 'default_server'
+    dest: /etc/nginx/sites-enabled/default_server
+    mode: 0644
+    group: root
+    owner: root
+  become: true
+  when: not default_server
+

data/ansible/roles/nginx-rails/templates/default_server

@@ -0,0 +1,5 @@
+server {
+    listen      80 default_server;
+    server_name _;
+    return 444;
+}

data/ansible/roles/nginx/tasks/main.yml

@@ -28,3 +28,7 @@
 - name: Restart nginx
   action: service name=nginx state=restarted
   become: true
+
+- name: Nginx is installed
+  set_fact:
+    nginx_installed: true

data/ansible/roles/papertrail/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
   - name: Install remote_syslog from papertrail
-    command: wget -O /tmp/remote_syslog.tar.gz https://github.com/papertrail/remote_syslog2/releases/download/v0.13/remote_syslog_linux_amd64.tar.gz  creates=/usr/bin/remote_syslog
+    command: wget -O /tmp/remote_syslog.tar.gz https://github.com/papertrail/remote_syslog2/releases/download/v0.20/remote_syslog_linux_amd64.tar.gz  creates=/usr/bin/remote_syslog
 
   - command: tar xzf /tmp/remote_syslog.tar.gz chdir=/tmp/ creates=/usr/bin/remote_syslog
 
@@ -10,7 +10,7 @@
   - file: path=/usr/bin/remote_syslog owner=root group=root mode=0755
     become: true
 
-  - command: wget -O /etc/init.d/remote_syslog https://raw.githubusercontent.com/papertrail/remote_syslog2/v0.13/examples/remote_syslog.init.d creates=/etc/init.d/remote_syslog
+  - command: wget -O /etc/init.d/remote_syslog https://raw.githubusercontent.com/papertrail/remote_syslog2/v0.20/examples/remote_syslog.init.d creates=/etc/init.d/remote_syslog
     become: true
 
   - file: path=/etc/init.d/remote_syslog owner=root group=root mode=0755
@@ -20,8 +20,12 @@
 
   - file: path=/tmp/remote_syslog.tar.gz state=absent
 
-  - service: name=remote_syslog state=restarted enabled=yes
+  - name: Create /etc/log_files
+    template: src=log_files.yml dest=/etc/log_files.yml owner=root group=root mode=0644
     become: true
 
-  - template: src=log_files.yml dest=/etc/log_files.yml owner=root group=root mode=0644
+  - name: Restart rsyslog
+    service: name=remote_syslog state=restarted enabled=yes
     become: true
+
+

data/ansible/roles/papertrail/templates/log_files.yml

@@ -1,7 +1,14 @@
 # Variables: papertrail_host, papertrail_port
 files:
-  - /var/log/apache2/error.log
   - /u/apps/{{project_name}}/shared/log/{{rails_env}}.log
+{% if nginx_installed is defined %}
+  - /var/log/nginx/error.log
+{% endif %}
+{% if apache_installed is defined %}
+  - /var/log/apache2/error.log
+{% endif %}
+
 destination:
   host: {{papertrail_host}}
   port: {{papertrail_port}}
+  protocol: tls

data/ansible/roles/postgresql-client/tasks/main.yml

@@ -47,3 +47,29 @@
     - postgresql
     - db
     - deps
+
+- name: Grab Psql version
+  shell: psql --version
+  register: stats_psql_version
+  when: send_stats == true and stats_url is defined and stats_api_key is defined
+  tags:
+    - maintenance
+    - stats
+
+- name: Send Psql stats to URL
+  uri:
+    url: "{{stats_url}}"
+    method: POST
+    headers:
+      X-API-Version: 1
+      X-Client-Api-key: "{{stats_api_key}}"
+    body_format: json
+    body:
+      client_stat:
+        key: psql_version
+        value: "{{stats_psql_version.stdout}}"
+        hostname: "{{hostname}}"
+  when: send_stats == true and stats_url is defined and stats_api_key is defined
+  tags:
+    - maintenance
+    - stats

data/ansible/roles/postgresql/meta/main.yml

@@ -1,7 +1,2 @@
 ---
-  dependencies:
-    - {
-      role: zenoamaro.postgresql,
-      become: true,
-      notify: postgresql restart
-    }
+  dependencies:

data/ansible/roles/postgresql/tasks/main.yml

@@ -1,6 +1,64 @@
 ---
   - set_fact: postgresql_installed="true"
 
+  - name: Adding APT repository key
+    become: yes
+    apt_key:
+      id: ACCC4CF8
+      url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
+    tags:
+      - postgresql
+      - db
+      - repo
+
+  - name: Add PostgreSQL official APT repository
+    become: yes
+    apt_repository:
+      repo: "deb http://apt.postgresql.org/pub/repos/apt/ {{ansible_distribution_release}}-pgdg main"
+    tags:
+      - postgresql
+      - db
+      - repo
+
+  - name: Install PostgreSQL
+    become: yes
+    apt:
+      name: "{{item}}"
+      state: present
+      update_cache: yes
+      cache_valid_time: 3600
+    with_items:
+      - "postgresql-{{postgresql_version}}"
+      - "postgresql-client-{{postgresql_version}}"
+      - "libpq-dev"
+    tags:
+      - postgresql
+      - db
+      - deps
+
+  - name: "Detect python3"
+    shell: "which python3"
+    register: is_python3
+
+  - name: Ensure pip is installed (python3)
+    when: is_python3 is succeeded
+    apt:
+      name: python3-pip
+      state: present
+      update_cache: yes
+
+  - name: Install psycopg2 (python3)
+    when: is_python3 is succeeded
+    become: yes
+    command: "pip3 install psycopg2"
+
+  - name: Install psycopg2 (python2)
+    when: is_python3 is failed
+    become: yes
+    apt:
+      name: python-psycopg2
+      state: latest
+
   - name: Create postgresql user
     postgresql_user:
       name: "{{database_user}}"
@@ -38,3 +96,29 @@
 
   - include: backups.yml
     become: true
+
+  - name: Grab Psql version
+    shell: psql --version
+    register: stats_psql_version
+    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    tags:
+      - maintenance
+      - stats
+
+  - name: Send Psql stats to URL
+    uri:
+      url: "{{stats_url}}"
+      method: POST
+      headers:
+        X-API-Version: 1
+        X-Client-Api-key: "{{stats_api_key}}"
+      body_format: json
+      body:
+        client_stat:
+          key: psql_version
+          value: "{{stats_psql_version.stdout}}"
+          hostname: "{{hostname}}"
+    when: send_stats == true and stats_url is defined and stats_api_key is defined
+    tags:
+      - maintenance
+      - stats

data/ansible/roles/sidekiq/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-  sidekiq_concurrency: 5
+  sidekiq_concurrency: 10

data/ansible/roles/sidekiq/templates/sidekiq-monit-rc

@@ -1,4 +1,4 @@
 check process sidekiq
   with pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid
-  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
+  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} -c {{sidekiq_concurrency}} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
   stop program = "/bin/su - deploy -c 'kill -s TERM `cat /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid`'" with timeout 30 seconds

data/ansible/roles/zenoamaro.postgresql/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 
-postgresql_version: 9.4
+# BS -- Commenting this out to force people to
+# postgresql_version: 9.4
 
 # This will be the main admin user, which is only allowed to connect
 # from localhost, mainly for provisioning, maintenance and scripts.

data/lib/subspace/version.rb

@@ -1,3 +1,3 @@
 module Subspace
-  VERSION = "2.2.3"
+  VERSION = "2.4.0"
 end

data/subspace.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: subspace
 version: !ruby/object:Gem::Version
-  version: 2.2.3
+  version: 2.4.0
 platform: ruby
 authors:
 - Brian Samson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,9 @@ files:
 - ansible/roles/delayed_job/tasks/main.yml
 - ansible/roles/delayed_job/templates/delayed-job-monit-rc
 - ansible/roles/letsencrypt/defaults/main.yml
+- ansible/roles/letsencrypt/tasks/legacy.yml
 - ansible/roles/letsencrypt/tasks/main.yml
+- ansible/roles/letsencrypt/tasks/modern.yml
 - ansible/roles/letsencrypt_dns/defaults/main.yml
 - ansible/roles/letsencrypt_dns/tasks/main.yml
 - ansible/roles/logrotate/LICENSE
@@ -179,6 +181,7 @@ files:
 - ansible/roles/nginx-rails/templates/_asset_cors.conf
 - ansible/roles/nginx-rails/templates/_rails.conf
 - ansible/roles/nginx-rails/templates/_upstream.conf
+- ansible/roles/nginx-rails/templates/default_server
 - ansible/roles/nginx-rails/templates/nginx-project
 - ansible/roles/nginx-rails/templates/nginx-project-ssl
 - ansible/roles/nginx/defaults/main.yml
@@ -303,7 +306,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Ansible-based server provisioning for rails projects