subspace 2.3.0 → 2.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c1bb8ea735bdf17e01f7941b5ca6d19c33cdb7c6e28b9715001eaa4f6c357e6
-  data.tar.gz: bdd11bf25939821875624dfcc8124247f7f1f9eadeae1e271e5bdab7a26799a5
+  metadata.gz: 453532a828c7f913ca85866e4e117daef85e0b21dd86092e734abb99ddcaad3a
+  data.tar.gz: e375411d365d7c7decaec797d648766187d2e54512d4226929ebd279686464ba
 SHA512:
-  metadata.gz: c1b5a144b9dfa225858940a1fde22d66790183f4fd10b26fcc9920b0a8e609555760985e0e476d201cebb88c466881f8d3397a95447409a2e4ed64b54fafa4b7
-  data.tar.gz: 5bdb929682b534293f6f91a7a6deab520ccbe8e7711bf2253c713e2bb7391fef271133aee3cf171317e8ecb8a106bdb0a7f58921b465883099ccadff75bb905e
+  metadata.gz: 74b2ba76f056e920cb53e88ebb1b5a8eb3c2e3830158bca9cda71661417b3506a99c27ff4d7d52ebf770f9a22102b86ed712cf1065753fb9d10d855a4c209882
+  data.tar.gz: a11f6ecb4634140f1175415ea3c78160bad5a2e21691b7e889359d23b8099fe49643bbda0901af6a92f7dff182590562c5a7a9b64dfa9accddeef375db068d87

data/CHANGELOG.md

@@ -10,6 +10,34 @@ This project attempts to follow [semantic versioning](https://semver.org/)
   * Not working on OSX - macs don't read from /etc/profile.d/
   * Stops showing color if you `sudo su`
 
+## 2.4.1
+  * Allow extra nginx options via extra_nginx_config eg:
+    ```
+    extra_nginx_config: |
+      proxy_http_version 1.1;
+      chunked_transfer_encoding off;
+      proxy_buffering off;
+      proxy_cache off;
+    ```
+  * Add keepalive_timeout for nginx
+
+## 2.4
+  Lots of modifications for ubuntu 20.04, which has python3 as a default
+
+  * Change letsencrypt to pull from apt instead of build from source (backwards compatible)
+  * Change postgres to a cleaner install and deprecate the old zenoamaro role
+  * postgresql_version is now a required variable and no longer defaults to 9.4
+  * Better detection of web servers
+
+## 2.3.3
+  * Tweak the way that different roles are detected to be more reliable
+
+## 2.3.2
+  * Update papertrail to latest version of remote_syslog2 and add support for nginx logs
+
+## 2.3.1
+  * Sidekiq concurrency actually works
+
 ## 2.3.0
   * Grab linux kernel to send as stats
   * Grab psql version to send as stats

data/README.md

@@ -291,6 +291,10 @@ Configures nginx to look at localhost:9292 for the socket/backend connection.  I
 defaults are here, we'll probably add more:
 
     client_max_body_size: 4G
+    ssl_force_redirect: true
+    default_server: true
+    keepalive_timeout: 10
+    extra_nginx_config: ""
 
 Optional variables:
 
@@ -300,6 +304,12 @@ Optional variables:
     nginx_proxy_read_timeout: Set [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). This is in seconds. You probably only want to change this if using rack-timeout (although I may be wrong). If using rack-timeout, it should be slightly higher than the rack-timeout timeout. I'm doing 5 seconds higher, but that was arbitrarily chosen.
 
     ssl_force_redirect: redirect all HTTP traffic to HTTPS on the same host.  Defaults to true and only applies if ssl_enabled is also true.
+    extra_nginx_config: anything else you want to configure in the main nginx config block, formatted like:
+    extra_nginx_config: |
+      proxy_http_version 1.1;
+      chunked_transfer_encoding off;
+      proxy_buffering off;
+      proxy_cache off;
 
 ## nodejs
 

data/ansible/roles/apache/tasks/main.yml

@@ -52,3 +52,7 @@
       state: link
     become: true
     notify: apache restart
+
+  - name: Apache is installed
+    set_fact:
+      apache_installed: true

data/ansible/roles/collectd/tasks/main.yml

@@ -47,7 +47,7 @@
       dest: /etc/collectd/collectd.conf.d/apache2.conf
     become: true
     notify: restart collectd
-    when: "'apache' in role_names"
+    when: apache_installed is defined
 
   - name: create puma config
     template:
@@ -70,7 +70,7 @@
       dest: /etc/collectd/collectd.conf.d/nginx.conf
     become: true
     notify: restart collectd
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
 
   - name: create rails_lograge config
     template:

data/ansible/roles/common/templates/motd

@@ -4,7 +4,7 @@ This server brought to you by:
 \___ \| | | | '_ \___ \| '_ \ / _` |/ __/ _ \
  ___) | |_| | |_) |__) | |_) | (_| | (_|  __/
 |____/ \__,_|_.__/____/| .__/ \__,_|\___\___|
-                       |_|             v2.3.0
+                       |_|             v2.4.1
 ~~~ https://github.com/tenforwardconsulting/subspace ~~~
 
 If you need to make configuration changes to the server, please modify the

data/ansible/roles/letsencrypt/tasks/legacy.yml

@@ -0,0 +1,44 @@
+---
+  - name: Install certbot dependencies
+    become: true
+    apt:
+      pkg: "{{item}}"
+      state: present
+    with_items:
+      - augeas-lenses
+      - ca-certificates
+      - dialog
+      - gcc
+      - libaugeas0
+      - libffi-dev
+      - libpython-dev
+      - libpython2.7-dev
+      - libssl-dev
+      - python
+      - python-dev
+      - python-setuptools
+      - python-virtualenv
+      - python2.7
+      - python2.7-dev
+
+  - name: "Create certbot dir"
+    become: true
+    file:
+      path: "{{certbot_dir}}"
+      state: directory
+      mode: 0755
+
+  - name: "Set certbot binary"
+    set_fact:
+      certbot_bin: "{{certbot_dir}}/certbot_auto"
+
+  - name: Get certbot
+    become: true
+    get_url:
+      url: "https://dl.eff.org/certbot-auto"
+      dest: "{{certbot_bin}}"
+      mode: a+x
+
+
+
+

data/ansible/roles/letsencrypt/tasks/main.yml

@@ -1,38 +1,32 @@
 ---
-  - name: Install certbot dependencies
+  - name: Ensure nginx is installed (first time)
     become: true
     apt:
-      pkg: "{{item}}"
+      pkg: nginx
       state: present
-    with_items:
-      - augeas-lenses
-      - ca-certificates
-      - dialog
-      - gcc
-      - libaugeas0
-      - libffi-dev
-      - libpython-dev
-      - libpython2.7-dev
-      - libssl-dev
-      - python
-      - python-dev
-      - python-setuptools
-      - python-virtualenv
-      - python2.7-dev
-
-  - name: "Create certbot dir"
-    become: true
-    file:
-      path: "{{certbot_dir}}"
-      state: directory
-      mode: 0755
+    when: "'nginx' in role_names"
 
-  - name: Get certbot
+  - name: Attempt to install certbot from APT
     become: true
-    get_url:
-      url: "https://dl.eff.org/certbot-auto"
-      dest: "{{certbot_dir}}/certbot-auto"
-      mode: a+x
+    ignore_errors: true
+    apt:
+      pkg: certbox
+      state: present
+
+  - name: "Detect if certbot was installed via APT"
+    shell: dpkg-query -W 'certbot'
+    ignore_errors: true
+    register: apt_certbot
+
+  - name: "Modern Letsencrypt Installation (py3, apt version)"
+    include_tasks: modern.yml
+    when: apt_certbot is succeeded
+
+  - name: "Legacy Letsencrypt Installation (py2, from source)"
+    include_tasks: legacy.yml
+    when: apt_certbot is failed
+
+# Post install configuration
 
   - name: shutdown webserver for standalone mode
     debug: msg="Shutdown webserver"
@@ -50,21 +44,21 @@
   - name: Run default
     when: le_ssl_certs is not defined
     become: true
-    command: "{{certbot_dir}}/certbot-auto certonly --email {{letsencrypt_email}} --domains {{([server_name] + server_aliases) | join(',')}} --standalone --agree-tos --expand --non-interactive"
+    command: "{{certbot_bin}} certonly --email {{letsencrypt_email}} --domains {{([server_name] + server_aliases) | join(',')}} --standalone --agree-tos --expand --non-interactive"
 
   - name: Generate SSL Certificates
     become: true
     with_items: "{{le_ssl_certs|default([])}}"
-    command: "{{certbot_dir}}/certbot-auto certonly --email {{letsencrypt_email}} --domains {{item.domains | join(',')}} --cert-name {{item.cert_name}} --standalone --agree-tos --expand --non-interactive"
+    command: "{{certbot_bin}} certonly --email {{letsencrypt_email}} --domains {{item.domains | join(',')}} --cert-name {{item.cert_name}} --standalone --agree-tos --expand --non-interactive"
 
   - name: Update nginx default options
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
     get_url:
       url: https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf
       dest: /etc/letsencrypt/options-ssl-nginx.conf
 
   - name: Update apache default options
-    when: "'apache' in role_names"
+    when: apache_installed is defined
     get_url:
       url: https://raw.githubusercontent.com/certbot/certbot/master/certbot-apache/certbot_apache/options-ssl-apache.conf
       dest: /etc/letsencrypt/options-ssl-apache.conf
@@ -82,20 +76,20 @@
 
   - name: Setup cron job to auto renew
     become: true
-    when: "'apache' in role_names"
+    when: apache_installed is defined
     cron:
       name: Auto-renew SSL
-      job: "{{certbot_dir}}/certbot-auto renew --no-self-upgrade --apache >> /var/log/cron.log 2>&1"
+      job: "{{certbot_bin}} renew --no-self-upgrade --apache >> /var/log/cron.log 2>&1"
       hour: "0"
       minute: "33"
       state: present
 
   - name: Setup cron job to auto renew
     become: true
-    when: "'nginx' in role_names"
+    when: nginx_installed is defined
     cron:
       name: Auto-renew SSL
-      job: "{{certbot_dir}}/certbot-auto renew --no-self-upgrade --nginx >> /var/log/cron.log 2>&1"
+      job: "{{certbot_bin}} renew --no-self-upgrade --nginx >> /var/log/cron.log 2>&1"
       hour: "0"
       minute: "33"
-      state: present
+      state: present

data/ansible/roles/letsencrypt/tasks/modern.yml

@@ -0,0 +1,13 @@
+---
+  - name: Install ca-certificates
+    become: true
+    apt:
+      pkg: "{{item}}"
+      state: present
+    with_items:
+      - ca-certificates
+
+  - name: "Set certbot binary"
+    set_fact:
+      certbot_bin: "certbot"
+

data/ansible/roles/nginx-rails/defaults/main.yml

@@ -2,3 +2,5 @@
   client_max_body_size: 4G
   ssl_force_redirect: true
   default_server: true
+  keepalive_timeout: 10
+  extra_nginx_config: ""

data/ansible/roles/nginx-rails/tasks/main.yml

@@ -25,3 +25,14 @@
     dest: /etc/nginx/sites-enabled/{{project_name}}-ssl
     state: "{{ (ssl_enabled and nginx_ssl_config is defined) | ternary('link', 'absent') }}"
   become: true
+
+- name: Enable a default server if one is not defined in the app
+  template:
+    src: 'default_server'
+    dest: /etc/nginx/sites-enabled/default_server
+    mode: 0644
+    group: root
+    owner: root
+  become: true
+  when: not default_server
+

data/ansible/roles/nginx-rails/templates/_rails.conf

@@ -15,6 +15,7 @@
     {% if nginx_proxy_read_timeout is defined %}
     proxy_read_timeout {{nginx_proxy_read_timeout}};
     {% endif %}
+    {{ extra_nginx_config | indent( width=4 ) }}
   }
 
   {% if asset_cors_allow_origin is defined %}
@@ -29,5 +30,5 @@
     root /opt/subspace;
   }
   client_max_body_size {{client_max_body_size}};
-  keepalive_timeout 10;
+  keepalive_timeout {{keepalive_timeout}};
 

data/ansible/roles/nginx-rails/templates/default_server

@@ -0,0 +1,5 @@
+server {
+    listen      80 default_server;
+    server_name _;
+    return 444;
+}

data/ansible/roles/nginx/tasks/main.yml

@@ -28,3 +28,7 @@
 - name: Restart nginx
   action: service name=nginx state=restarted
   become: true
+
+- name: Nginx is installed
+  set_fact:
+    nginx_installed: true

data/ansible/roles/papertrail/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
   - name: Install remote_syslog from papertrail
-    command: wget -O /tmp/remote_syslog.tar.gz https://github.com/papertrail/remote_syslog2/releases/download/v0.13/remote_syslog_linux_amd64.tar.gz  creates=/usr/bin/remote_syslog
+    command: wget -O /tmp/remote_syslog.tar.gz https://github.com/papertrail/remote_syslog2/releases/download/v0.20/remote_syslog_linux_amd64.tar.gz  creates=/usr/bin/remote_syslog
 
   - command: tar xzf /tmp/remote_syslog.tar.gz chdir=/tmp/ creates=/usr/bin/remote_syslog
 
@@ -10,7 +10,7 @@
   - file: path=/usr/bin/remote_syslog owner=root group=root mode=0755
     become: true
 
-  - command: wget -O /etc/init.d/remote_syslog https://raw.githubusercontent.com/papertrail/remote_syslog2/v0.13/examples/remote_syslog.init.d creates=/etc/init.d/remote_syslog
+  - command: wget -O /etc/init.d/remote_syslog https://raw.githubusercontent.com/papertrail/remote_syslog2/v0.20/examples/remote_syslog.init.d creates=/etc/init.d/remote_syslog
     become: true
 
   - file: path=/etc/init.d/remote_syslog owner=root group=root mode=0755
@@ -20,8 +20,12 @@
 
   - file: path=/tmp/remote_syslog.tar.gz state=absent
 
-  - service: name=remote_syslog state=restarted enabled=yes
+  - name: Create /etc/log_files
+    template: src=log_files.yml dest=/etc/log_files.yml owner=root group=root mode=0644
     become: true
 
-  - template: src=log_files.yml dest=/etc/log_files.yml owner=root group=root mode=0644
+  - name: Restart rsyslog
+    service: name=remote_syslog state=restarted enabled=yes
     become: true
+
+

data/ansible/roles/papertrail/templates/log_files.yml

@@ -1,7 +1,14 @@
 # Variables: papertrail_host, papertrail_port
 files:
-  - /var/log/apache2/error.log
   - /u/apps/{{project_name}}/shared/log/{{rails_env}}.log
+{% if nginx_installed is defined %}
+  - /var/log/nginx/error.log
+{% endif %}
+{% if apache_installed is defined %}
+  - /var/log/apache2/error.log
+{% endif %}
+
 destination:
   host: {{papertrail_host}}
   port: {{papertrail_port}}
+  protocol: tls

data/ansible/roles/postgresql/meta/main.yml

@@ -1,7 +1,2 @@
 ---
-  dependencies:
-    - {
-      role: zenoamaro.postgresql,
-      become: true,
-      notify: postgresql restart
-    }
+  dependencies:

data/ansible/roles/postgresql/tasks/main.yml

@@ -1,6 +1,64 @@
 ---
   - set_fact: postgresql_installed="true"
 
+  - name: Adding APT repository key
+    become: yes
+    apt_key:
+      id: ACCC4CF8
+      url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
+    tags:
+      - postgresql
+      - db
+      - repo
+
+  - name: Add PostgreSQL official APT repository
+    become: yes
+    apt_repository:
+      repo: "deb http://apt.postgresql.org/pub/repos/apt/ {{ansible_distribution_release}}-pgdg main"
+    tags:
+      - postgresql
+      - db
+      - repo
+
+  - name: Install PostgreSQL
+    become: yes
+    apt:
+      name: "{{item}}"
+      state: present
+      update_cache: yes
+      cache_valid_time: 3600
+    with_items:
+      - "postgresql-{{postgresql_version}}"
+      - "postgresql-client-{{postgresql_version}}"
+      - "libpq-dev"
+    tags:
+      - postgresql
+      - db
+      - deps
+
+  - name: "Detect python3"
+    shell: "which python3"
+    register: is_python3
+
+  - name: Ensure pip is installed (python3)
+    when: is_python3 is succeeded
+    apt:
+      name: python3-pip
+      state: present
+      update_cache: yes
+
+  - name: Install psycopg2 (python3)
+    when: is_python3 is succeeded
+    become: yes
+    command: "pip3 install psycopg2"
+
+  - name: Install psycopg2 (python2)
+    when: is_python3 is failed
+    become: yes
+    apt:
+      name: python-psycopg2
+      state: latest
+
   - name: Create postgresql user
     postgresql_user:
       name: "{{database_user}}"

data/ansible/roles/sidekiq/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-  sidekiq_concurrency: 5
+  sidekiq_concurrency: 10

data/ansible/roles/sidekiq/templates/sidekiq-monit-rc

@@ -1,4 +1,4 @@
 check process sidekiq
   with pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid
-  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
+  start program = "/bin/su - deploy -c 'cd /u/apps/{{project_name}}/current && bundle exec sidekiq --queue {{hostname}} {{ job_queues | map('regex_replace',  '^(.*)$', '--queue \\1') | join(' ') }} -c {{sidekiq_concurrency}} --pidfile /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid --environment {{rails_env}} --logfile /u/apps/{{project_name}}/shared/log/sidekiq.log --daemon'" with timeout 30 seconds
   stop program = "/bin/su - deploy -c 'kill -s TERM `cat /u/apps/{{project_name}}/shared/tmp/pids/sidekiq.pid`'" with timeout 30 seconds

data/ansible/roles/zenoamaro.postgresql/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 
-postgresql_version: 9.4
+# BS -- Commenting this out to force people to
+# postgresql_version: 9.4
 
 # This will be the main admin user, which is only allowed to connect
 # from localhost, mainly for provisioning, maintenance and scripts.

data/lib/subspace/version.rb

@@ -1,3 +1,3 @@
 module Subspace
-  VERSION = "2.3.0"
+  VERSION = "2.4.1"
 end

data/subspace.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: subspace
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.1
 platform: ruby
 authors:
 - Brian Samson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-28 00:00:00.000000000 Z
+date: 2020-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,9 @@ files:
 - ansible/roles/delayed_job/tasks/main.yml
 - ansible/roles/delayed_job/templates/delayed-job-monit-rc
 - ansible/roles/letsencrypt/defaults/main.yml
+- ansible/roles/letsencrypt/tasks/legacy.yml
 - ansible/roles/letsencrypt/tasks/main.yml
+- ansible/roles/letsencrypt/tasks/modern.yml
 - ansible/roles/letsencrypt_dns/defaults/main.yml
 - ansible/roles/letsencrypt_dns/tasks/main.yml
 - ansible/roles/logrotate/LICENSE
@@ -179,6 +181,7 @@ files:
 - ansible/roles/nginx-rails/templates/_asset_cors.conf
 - ansible/roles/nginx-rails/templates/_rails.conf
 - ansible/roles/nginx-rails/templates/_upstream.conf
+- ansible/roles/nginx-rails/templates/default_server
 - ansible/roles/nginx-rails/templates/nginx-project
 - ansible/roles/nginx-rails/templates/nginx-project-ssl
 - ansible/roles/nginx/defaults/main.yml