subspace 0.1.1 → 3.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/.ruby-version +1 -1
  4. data/.travis.yml +1 -1
  5. data/CHANGELOG.md +380 -0
  6. data/README.md +511 -9
  7. data/TODO +34 -0
  8. data/UPGRADING.md +10 -0
  9. data/ansible/playbooks/local_template.yml +13 -0
  10. data/ansible/playbooks/maintenance_mode.yml +5 -0
  11. data/ansible/roles/alienvault/tasks/main.yml +29 -0
  12. data/ansible/roles/apache/defaults/main.yml +2 -5
  13. data/ansible/roles/apache/handlers/main.yml +9 -1
  14. data/ansible/roles/apache/tasks/main.yml +37 -9
  15. data/ansible/roles/apache/templates/server_status.conf +6 -0
  16. data/ansible/roles/apache-rails/tasks/main.yml +26 -0
  17. data/ansible/roles/apache-rails/templates/_canonical_domain.conf +10 -0
  18. data/ansible/roles/apache-rails/templates/_rails.conf +23 -0
  19. data/ansible/roles/apache-rails/templates/project-ssl.conf +23 -0
  20. data/ansible/roles/apache-rails/templates/project.conf +5 -0
  21. data/ansible/roles/collectd/defaults/main.yml +3 -0
  22. data/ansible/roles/collectd/handlers/main.yml +3 -0
  23. data/ansible/roles/collectd/tasks/main.yml +82 -0
  24. data/ansible/roles/collectd/templates/apache2.conf +6 -0
  25. data/ansible/roles/collectd/templates/delayed_job_postgres.conf +21 -0
  26. data/ansible/roles/collectd/templates/df.conf +16 -0
  27. data/ansible/roles/collectd/templates/graphite.conf +14 -0
  28. data/ansible/roles/collectd/templates/hostname.conf +2 -0
  29. data/ansible/roles/collectd/templates/nginx.conf +28 -0
  30. data/ansible/roles/collectd/templates/nodejs-process.conf +3 -0
  31. data/ansible/roles/collectd/templates/puma-process.conf +3 -0
  32. data/ansible/roles/collectd/templates/rails_lograge.conf +31 -0
  33. data/ansible/roles/common/defaults/main.yml +2 -1
  34. data/ansible/roles/common/files/sudoers-service +1 -0
  35. data/ansible/roles/common/tasks/main.yml +307 -19
  36. data/ansible/roles/common/tasks/no_swap.yml +26 -0
  37. data/ansible/roles/common/tasks/swap.yml +34 -8
  38. data/ansible/roles/common/templates/motd +13 -0
  39. data/ansible/roles/common/templates/motd2 +24 -0
  40. data/ansible/roles/common/templates/terminalcolor +25 -0
  41. data/ansible/roles/delayed_job/README.md +2 -4
  42. data/ansible/roles/delayed_job/defaults/main.yml +0 -2
  43. data/ansible/roles/delayed_job/handlers/main.yml +1 -1
  44. data/ansible/roles/delayed_job/tasks/main.yml +22 -29
  45. data/ansible/roles/delayed_job/templates/delayed-job-monit-rc +6 -5
  46. data/ansible/roles/delayed_job/templates/delayed-job-systemd.service +33 -0
  47. data/ansible/roles/gem-patch-report/README.md +137 -0
  48. data/ansible/roles/gem-patch-report/defaults/main.yml +2 -0
  49. data/ansible/roles/gem-patch-report/tasks/main.yml +89 -0
  50. data/ansible/roles/gem-patch-report/templates/gem-patch-report.sh.j2 +114 -0
  51. data/ansible/roles/letsencrypt/defaults/main.yml +11 -1
  52. data/ansible/roles/letsencrypt/tasks/legacy.yml +44 -0
  53. data/ansible/roles/letsencrypt/tasks/main.yml +72 -76
  54. data/ansible/roles/letsencrypt/tasks/modern.yml +39 -0
  55. data/ansible/roles/logrotate/LICENSE +27 -0
  56. data/ansible/roles/logrotate/README.md +70 -0
  57. data/ansible/roles/logrotate/defaults/main.yml +2 -0
  58. data/ansible/roles/logrotate/meta/main.yml +18 -0
  59. data/ansible/roles/logrotate/tasks/main.yml +15 -0
  60. data/ansible/roles/logrotate/templates/logrotate.d.j2 +16 -0
  61. data/ansible/roles/memcache/defaults/main.yml +2 -0
  62. data/ansible/roles/memcache/tasks/main.yml +16 -1
  63. data/ansible/roles/mysql/meta/main.yml +1 -1
  64. data/ansible/roles/mysql/tasks/main.yml +2 -2
  65. data/ansible/roles/mysql2_gem/meta/main.yml +1 -1
  66. data/ansible/roles/mysql2_gem/tasks/main.yml +1 -1
  67. data/ansible/roles/newrelic/tasks/main.yml +8 -1
  68. data/ansible/roles/newrelic-infra/defaults/main.yml +2 -0
  69. data/ansible/roles/newrelic-infra/handlers/main.yml +5 -0
  70. data/ansible/roles/newrelic-infra/tasks/main.yml +38 -0
  71. data/ansible/roles/newrelic-infra/templates/logs.yml.j2 +5 -0
  72. data/ansible/roles/newrelic-infra/templates/newrelic-infra.yml.j2 +2 -0
  73. data/ansible/roles/nginx/defaults/main.yml +4 -0
  74. data/ansible/roles/nginx/handlers/main.yml +12 -0
  75. data/ansible/roles/nginx/tasks/main.yml +113 -14
  76. data/ansible/roles/nginx/templates/nginx.conf +29 -0
  77. data/ansible/roles/nginx/templates/status +10 -0
  78. data/ansible/roles/nginx-maintenance/tasks/main.yml +33 -0
  79. data/ansible/roles/nginx-rails/defaults/main.yml +6 -0
  80. data/ansible/roles/nginx-rails/tasks/main.yml +54 -0
  81. data/ansible/roles/nginx-rails/templates/_asset_cors.conf +11 -0
  82. data/ansible/roles/nginx-rails/templates/_rails.conf +35 -0
  83. data/ansible/roles/nginx-rails/templates/_upstream.conf +11 -0
  84. data/ansible/roles/nginx-rails/templates/default_server +5 -0
  85. data/ansible/roles/nginx-rails/templates/nginx-project +9 -0
  86. data/ansible/roles/nginx-rails/templates/nginx-project-ssl +23 -0
  87. data/ansible/roles/nodejs/tasks/main.yml +33 -0
  88. data/ansible/roles/papertrail/tasks/main.yml +23 -4
  89. data/ansible/roles/papertrail/templates/log_files.yml +8 -1
  90. data/ansible/roles/postgis/.gitignore +3 -0
  91. data/ansible/roles/postgis/CHANGELOG.md +24 -0
  92. data/ansible/roles/postgis/LICENSE +202 -0
  93. data/ansible/roles/postgis/README.md +12 -0
  94. data/ansible/roles/postgis/meta/main.yml +15 -0
  95. data/ansible/roles/postgis/tasks/main.yml +20 -0
  96. data/ansible/roles/postgresql/README.md +15 -0
  97. data/ansible/roles/postgresql/defaults/main.yml +3 -0
  98. data/ansible/roles/postgresql/handlers/main.yml +1 -1
  99. data/ansible/roles/postgresql/meta/main.yml +1 -6
  100. data/ansible/roles/postgresql/tasks/backups.yml +8 -6
  101. data/ansible/roles/postgresql/tasks/main.yml +89 -9
  102. data/ansible/roles/postgresql/templates/backup.sh +17 -2
  103. data/ansible/roles/postgresql-client/tasks/main.yml +91 -0
  104. data/ansible/roles/puma/tasks/main.yml +38 -19
  105. data/ansible/roles/puma/templates/puma-monit-rc +5 -0
  106. data/ansible/roles/puma/templates/puma-systemd.service +43 -0
  107. data/ansible/roles/puma/templates/puma-systemd.socket +14 -0
  108. data/ansible/roles/puma/templates/puma.rb +24 -12
  109. data/ansible/roles/rails/defaults/main.yml +1 -3
  110. data/ansible/roles/rails/tasks/main.yml +76 -18
  111. data/ansible/roles/rails/templates/application.yml +1 -1
  112. data/ansible/roles/rails/templates/database.yml +3 -3
  113. data/ansible/roles/redis/defaults/main.yml +2 -0
  114. data/ansible/roles/redis/tasks/main.yml +45 -3
  115. data/ansible/roles/resque/tasks/main.yml +14 -0
  116. data/ansible/roles/resque/templates/resque-monit-rc +4 -0
  117. data/ansible/roles/resque/templates/resque-systemd.service +54 -0
  118. data/ansible/roles/ruby-common/README.md +1 -1
  119. data/ansible/roles/ruby-common/defaults/main.yml +1 -0
  120. data/ansible/roles/ruby-common/tasks/main.yml +55 -13
  121. data/ansible/roles/sidekiq/README.md +1 -0
  122. data/ansible/roles/sidekiq/defaults/main.yml +2 -0
  123. data/ansible/roles/sidekiq/tasks/main.yml +15 -0
  124. data/ansible/roles/sidekiq/templates/sidekiq-monit-rc +4 -0
  125. data/ansible/roles/sidekiq/templates/sidekiq-systemd.service +63 -0
  126. data/ansible/roles/tailscale/defaults/main.yml +2 -0
  127. data/ansible/roles/tailscale/tasks/main.yml +40 -0
  128. data/ansible/roles/yarn/tasks/main.yml +19 -0
  129. data/ansible/roles/zenoamaro.postgresql/.gitignore +1 -0
  130. data/ansible/roles/zenoamaro.postgresql/.travis.yml +28 -0
  131. data/ansible/roles/zenoamaro.postgresql/LICENSE.md +21 -0
  132. data/ansible/roles/zenoamaro.postgresql/README.md +117 -0
  133. data/ansible/roles/zenoamaro.postgresql/Vagrantfile +62 -0
  134. data/ansible/roles/zenoamaro.postgresql/boxed.yml +26 -0
  135. data/ansible/roles/zenoamaro.postgresql/defaults/main.yml +607 -0
  136. data/ansible/roles/zenoamaro.postgresql/handlers/main.yml +8 -0
  137. data/ansible/roles/zenoamaro.postgresql/inventory +19 -0
  138. data/ansible/roles/zenoamaro.postgresql/main.yml +19 -0
  139. data/ansible/roles/zenoamaro.postgresql/meta/.galaxy_install_info +1 -0
  140. data/ansible/roles/zenoamaro.postgresql/meta/main.yml +21 -0
  141. data/ansible/roles/zenoamaro.postgresql/tasks/configure.yml +55 -0
  142. data/ansible/roles/zenoamaro.postgresql/tasks/extensions.yml +49 -0
  143. data/ansible/roles/zenoamaro.postgresql/tasks/install.yml +51 -0
  144. data/ansible/roles/zenoamaro.postgresql/tasks/main.yml +5 -0
  145. data/ansible/roles/zenoamaro.postgresql/templates/environment +11 -0
  146. data/ansible/roles/zenoamaro.postgresql/templates/pg_ctl.conf +5 -0
  147. data/ansible/roles/zenoamaro.postgresql/templates/pg_hba.conf +112 -0
  148. data/ansible/roles/zenoamaro.postgresql/templates/pg_ident.conf +46 -0
  149. data/ansible/roles/zenoamaro.postgresql/templates/postgresql.conf +536 -0
  150. data/ansible/roles/zenoamaro.postgresql/test.yml +24 -0
  151. data/bin/console +0 -4
  152. data/exe/subspace +3 -84
  153. data/lib/subspace/cli.rb +168 -0
  154. data/lib/subspace/commands/ansible.rb +35 -0
  155. data/lib/subspace/commands/base.rb +94 -0
  156. data/lib/subspace/commands/bootstrap.rb +49 -0
  157. data/lib/subspace/commands/configure.rb +27 -0
  158. data/lib/subspace/commands/exec.rb +20 -0
  159. data/lib/subspace/commands/init.rb +144 -0
  160. data/lib/subspace/commands/inventory.rb +54 -0
  161. data/lib/subspace/commands/maintain.rb +16 -0
  162. data/lib/subspace/commands/maintenance_mode.rb +16 -0
  163. data/lib/subspace/commands/override.rb +21 -0
  164. data/lib/subspace/commands/provision.rb +14 -0
  165. data/lib/subspace/commands/secrets.rb +69 -0
  166. data/lib/subspace/commands/ssh.rb +33 -0
  167. data/lib/subspace/commands/terraform.rb +89 -0
  168. data/lib/subspace/configuration.rb +71 -0
  169. data/lib/subspace/inventory.rb +144 -0
  170. data/lib/subspace/version.rb +1 -1
  171. data/lib/subspace.rb +8 -1
  172. data/subspace.gemspec +14 -3
  173. data/template/provision.rb.erb +94 -0
  174. data/template/subspace/.gitignore +6 -0
  175. data/template/subspace/ansible.cfg.erb +17 -0
  176. data/template/subspace/group_vars/all.erb +28 -0
  177. data/template/subspace/group_vars/template.erb +26 -0
  178. data/template/subspace/inventory.yml.erb +12 -0
  179. data/template/subspace/playbook.yml.erb +18 -0
  180. data/template/subspace/secrets/template.erb +5 -0
  181. data/template/subspace/templates/application.yml.template +27 -0
  182. data/template/subspace/templates/authorized_keys.erb +1 -0
  183. data/template/subspace/terraform/.gitignore +2 -0
  184. data/template/subspace/terraform/template/main-oxenwagen.tf.erb +116 -0
  185. data/template/subspace/terraform/template/main-workhorse.tf.erb +41 -0
  186. data/template/subspace/terraformrc.erb +9 -0
  187. data/terraform/modules/s3_backend/README +2 -0
  188. data/terraform/modules/s3_backend/dynamodb.tf +1 -0
  189. data/terraform/modules/s3_backend/iam_user.tf +38 -0
  190. data/terraform/modules/s3_backend/main.tf +39 -0
  191. data/terraform/modules/s3_backend/state_bucket.tf +14 -0
  192. metadata +198 -29
  193. data/ansible/roles/apache/templates/project.conf +0 -25
  194. data/ansible/roles/aws/tasks/main.yml +0 -46
  195. data/ansible/roles/delayed_job/files/deploy-service +0 -2
  196. data/ansible/roles/delayed_job/files/monit-http.conf +0 -3
  197. data/ansible/roles/letsencrypt/templates/project-le-ssl.conf +0 -31
  198. data/ansible/roles/nginx/templates/nginx-project +0 -27
  199. data/ansible/roles/puma/templates/etc-init-puma-manager.conf +0 -31
  200. data/ansible/roles/puma/templates/etc-init-puma.conf +0 -69
  201. data/lib/subspace/railtie.rb +0 -9
  202. data/lib/tasks/provision.rake +0 -12
  203. data/template/provision/.gitignore +0 -1
  204. data/template/provision/ansible.cfg.erb +0 -5
  205. data/template/provision/group_vars/all.erb +0 -12
  206. data/template/provision/group_vars/template.erb +0 -14
  207. data/template/provision/host_vars/template.erb +0 -4
  208. data/template/provision/playbook.yml.erb +0 -26
  209. data/template/provision/vars/template.erb +0 -4
  210. /data/template/{provision → subspace}/hosts.erb +0 -0
data/README.md CHANGED
@@ -1,31 +1,525 @@
1
1
  # Subspace
2
2
 
3
+ [![Gem](https://img.shields.io/gem/v/subspace.svg)](https://rubygems.org/gems/subspace)
4
+
3
5
  Subspace is a rubygem meant to make provisioning as easy as Capistrano makes deploying.
4
6
 
5
7
  http://tvtropes.org/pmwiki/pmwiki.php/Main/SubspaceAnsible
6
8
 
9
+ It is powered by [Ansible](https://www.ansible.com/). Most of the roles require
10
+ you to [configure variables](https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html) that the role uses.
11
+
7
12
  ## Installation
8
13
 
14
+ First, install ansible (>2.0)
15
+
16
+ - OSX: `brew install ansible`
17
+ - Linux: `apt-get install ansible`
18
+
9
19
  Add this line to your application's Gemfile:
10
20
 
11
21
  ```ruby
12
22
  gem 'subspace'
13
23
  ```
14
24
 
15
- And then execute:
25
+ Or install it yourself from the command line: `$ gem install subspace`
26
+
27
+ ### Mitogen
28
+
29
+ [!CAUTION]
30
+ ***Mitogen is currently broken! It doesn't support the latest ansible version.***
31
+
32
+ Optionally, you can install a python/pip packaged called "Mitogen" which dramatically speeds up running ansible over ssh. See [Here](https://github.com/mitogen-hq/mitogen/blob/master/docs/ansible_detailed.rst) for details.
16
33
 
17
- $ bundle
34
+ pip install mitogen
18
35
 
19
- Or install it yourself as:
36
+ Subspace will try and detect if mitogen is present and use it can. If mitogen causes problems (sometimes it can cause problems depending on the system versions, and particaularly when brand new versions of anible come up and it hasn't updated), you can disable it:
20
37
 
21
- $ gem install subspace
38
+ DISABLE_MITOGEN=1 subspace provision env
22
39
 
23
40
  ## Usage
24
41
 
25
- subspace init
26
- cp path/to/.vault_pass config/provision/ # Or get it from another person
27
- cd config/provision && ansible-vault create vars/production.yml
28
- cd config/provision && ansible-vault create vars/dev.yml
42
+ ### `subspace init`
43
+
44
+ Initialize the project for subspace. Creates `config/subspace` with all necessary files.
45
+
46
+ Subspace 3 supports terraform. You will need to create an IAM user manually with administrative access to the target AWS environment for terraform.
47
+
48
+ #### Other Requirements
49
+
50
+ - [JQ](https://formulae.brew.sh/formula/jq)
51
+ - [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli)
52
+ - [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions)
53
+
54
+ #### Setup AWS Profile
55
+
56
+ `aws configure --profile profile_name`
57
+
58
+ Subspace expects the `profile_name` to be `subspace-{project name}`.
59
+
60
+ #### `init` Options
61
+
62
+ Check [`cli.rb init`](/lib/subspace/cli.rb#L35) for all available options when initializing a new subspace project.
63
+
64
+ ##### `--env`
65
+ The environment will default to `dev` unless you pass in `--env [env name]`
66
+
67
+ ### `subspace bootstrap <environment>`
68
+
69
+ Ensures the $HOME/.ssh directory is present and ensures python is installed.
70
+ Python is required to be installed on the remote server for the `provision`
71
+ command to work.
72
+
73
+ ### `subspace provision <environment>`
74
+
75
+ Runs the playbook at `config/provision/<environment.yml>`.
76
+
77
+ You can pass certain options through to the `ansible-playbook` command. See [the
78
+ provision command](lib/subspace/commands/provision.rb) for the current list.
79
+
80
+ At the time of this writing, we pass through the `ansible-playbook` "tags" and
81
+ "start-at-task" options. The tags option is probably the most useful.
82
+
83
+ e.g. To run only the alienvault tasks (all of which have been tagged with the
84
+ 'alienvault' tag): `subspace provision staging --tags=alienvault`
85
+
86
+ ### `subspace maintain <environment>`
87
+
88
+ Runs the playbook at `config/provision/<environment.yml>` but only the tasks tagged with "maintenance".
89
+
90
+ You can pass certain options through to the `ansible-playbook` command. See [the
91
+ maintain command](lib/subspace/commands/maintain.rb) for the current list.
92
+
93
+ At the time of this writing, we pass through the `ansible-playbook` "limit" option.
94
+
95
+ e.g. To run only on the host "prod-web1": `subspace maintain production --limit=prod-web1`
96
+
97
+ ### `subspace maintenance_mode <environment> --on`
98
+
99
+ This sets up nginx to return all requests as 503 and serve only the
100
+ static page at `/u/apps/{{project_name}}/current/public/maintenance.html`
101
+ which must be checked into your project and deployed to the server.
102
+
103
+ `--on` and `--off`, defaults to off.
104
+
105
+ Only works for hosts using the `nginx` role, but you can pass in your entire environment. Running it on worker servers won't hurt anything.
106
+
107
+ MUST be turned off manually by running `subspace maintenance_mode <environment> --off`, even a deploy will not disable maintenance mode.
108
+
109
+ #### Tagged roles
110
+
111
+ | Role | Tags | Comment |
112
+ | ---------- | ---------------- | -------------------------------------------------------------- |
113
+ | alienvault | alienvault | All tasks in the alienvault role have been tagged 'alienvault' |
114
+ | common | upgrade | runs apt-get update and apt-get upgrade |
115
+ | common | authorized\_keys | updates the authorized\_keys file for the deploy user |
116
+ | rails | appyml |
117
+ | monit | monit | All tasks in the monit role have been tagged 'monit' |
118
+
119
+ ### `subspace secrets <environment> [--edit] [--create]`
120
+
121
+ The `secrets` command will manage encrypted secrets for different environments. The default action is simply to show the secrets defined for an environment. Pass --edit to edit them in the system editor (vim, etc).
122
+
123
+ This uses `ansible-vault` under the hood and requires a vault password file. You will need to get the `.vault_pass` from from a teammate out of band (secrets.10fw.ne, 1password, sticky-note, etc), and put it into `config/provision/.vault_pass`
124
+
125
+ These secrets are used during provisioning to populate variables in a few different places:
126
+ 1. `config/application.yml`, which uses the `figaro` gem to manage environment variables in rails.
127
+ 2. `config/database.yml`, which handles the database connection password.
128
+
129
+
130
+ Subspace uses a template file in `config/provision/templates/application.yml.template` that contains environment variables for all environments. If you have non-secret variables that change based on the target server, you can simply put that in plaintext in the template file. This was designed so the configuration that is not secret is visible and version controlled, while the secret values are stored in the vault files for their environments.
131
+
132
+ NOTE: application.yml should be in the `.gitignore`, since subspace creates a new version on the server and symlinks it on top of whatever is checked in. You should make changes to the template file instead, which should be checked in to version control.
133
+
134
+
135
+ The default template created by `subspace init` looks like this:
136
+
137
+ ```
138
+ # These environment variables are applied to all environments, and can be secret or not
139
+
140
+ # This is secret and can be changed on all three environment easily by using subspace vars <env> --edit
141
+ SECRET_KEY_BASE: {{secret_key_base}}
142
+
143
+ # This is not secret, and is the same value for all environments
144
+ ENABLE_SOME_FEATURE: false
145
+
146
+ development:
147
+ INSECURE_VARIABLE: "this isn't secret"
148
+
149
+ staging:
150
+ INSECURE_VARIABLE: "but it changes"
151
+
152
+ production:
153
+ INSECURE_VARIABLE: "on different servers"
154
+
155
+ ```
156
+
157
+ You can also use this command to automatically create a local version of `config/application.yml` based on the template and encrypted secrets for a specific environment.
158
+
159
+ # Create a local copy of config/application.yml with the secrets encrypted in secrets/development.yml
160
+ $ subspace vars development --create
161
+
162
+ This can get you up and running quickly in development securely.
163
+
164
+ ## Procedure for updating on projects
165
+
166
+ When subspace is updated, you should also update it for projects that use it. If subspace is locked to a specific version in the a project's gemfile, you'll need to update that to the new release number, first. Make sure this won't introduce any breaking changes by [looking at the changelog](https://github.com/tenforwardconsulting/subspace/blob/master/CHANGELOG.md).
167
+
168
+ * `bundle update subspace`
169
+
170
+ Then,
171
+
172
+ * `subspace provision production`
173
+
174
+ If you get an error saying you need a vault password file, you need to get it from somoene on the team (see above). You might also need to update `ansible`.
175
+
176
+ You'll want to do this for each environment (ie: `subspace provision qa`, etc.). Best to start with staging and work your way up.
177
+
178
+ # Host configuration
179
+
180
+ We need to know some info about hosts, but not much. See the files for details, it's mostly the hostname and the user that can administer the system, eg `ubuntu` on AWS/ubuntu, `ec2-user`, or even `root` (not recommended, but used on linode/Digital Ocean)
181
+
182
+ # Role Configuration
183
+
184
+ This is a description of all the roles that are included by installing subspace, along with their configuration.
185
+
186
+ ## common
187
+
188
+ This role should almost always be there. It ties a bunch of stuff together, runs apt-get update or yum upgrade, sets hostnames, and generally makes the server sane.
189
+
190
+ project_name: my_project
191
+ swap_space: 512M
192
+ deploy_user: deploy
193
+
194
+ Note: we grant the deploy user limited sudo access to run `service xyz restart` and also add it to the `adm` group so it can view logs in `/var/log`.
195
+
196
+ # Roles
197
+
198
+ This is a description of all the roles that are included by installing subspace, along with their configuration.
199
+
200
+ ## alienvault
201
+
202
+ Fixes CVEs on remote servers by installing updated versions of packages
203
+ depending on the operating system version.
204
+
205
+ Each CVE fix is tagged by the CVE name so updates can be targeted to a specific
206
+ vulnerability.
207
+
208
+ Variables: None
209
+
210
+ ## apache
211
+
212
+ The most important file for an apache install is the "project.conf" file that gets created in `sites-available` and symlinked to `sites-enabled`. This is generated in a sensible way, but if you want to customize it you can do so by setting this variable to anything other than "project.conf":
213
+
214
+ apache_project_conf: my_custom_configuration.conf
215
+
216
+ Then place my_custom_configuration.conf in config/provision/templates/my_custom_configuration.conf. This will still get copied to the server as `sites-available/{project_name}.conf`
217
+
218
+ Apache also support canonicalizing the domain now, so if you alwyas want to redirect to WWW for example, simply add a variable:
219
+
220
+ canonical_domain: "www.example.com"
221
+
222
+ ## collectd
223
+
224
+ Collectd is a super useful daemon that grabs and reports statistics about a server's health. Adding this role will make your server start reporting to a [graphite](https://graphiteapp.org/) server that you specify, and you can make cool graphs and data feeds after that using something like [Grafana](https://grafana.com/)
225
+
226
+ graphite_host: graphite.example.com
227
+ graphite_port: "2003"
228
+
229
+ Aside from basic statistics like free memory, disk, load averages, etc, we have some custom things:
230
+
231
+ 1. If Postgres and delayed job are installed, it will collect stats on number of outstanding delayed jobs.
232
+ a. If you have pg on a different server or in RDS, you can set this manually:
233
+
234
+ collectd_pgdj: true
235
+
236
+ 2. If apache is installed, it will collect stats from the /server-status page
237
+ 3. If nginx is installed, it will collect stats from the "status port"
238
+ 4. (TODO) add something for pumas
239
+ 5. (TODO) add something for sidekiq
240
+ 6. If you're using our standard lograge format, you can enable lograge collection which will provide stats on request count and timers (db/view/total)
241
+
242
+ rails_lograge: true
243
+
244
+
245
+ ## common
246
+
247
+ ## delayed_job
248
+
249
+ Install monitoring and automatic startup for delayed job workers via monit. You MUST set the `job_queues` variable as follows:
250
+
251
+ job_queues:
252
+ - default
253
+ - mailers
254
+ - exports
255
+
256
+ If you want to have multiple workers for a single queue, just add the queue name
257
+ multiple times:
258
+
259
+ job_queues:
260
+ - default
261
+ - mailers
262
+ - exports
263
+ - exports
264
+ - exports
265
+
266
+ Please note that by default, delayed job does not set a queue (eg it uses the "null" queue). You MUST also add an initializer to your rails app where you set the default queue name to "default" (or some other queue). Otherwise, the named queue workers managed by this role will not process the "null" queue.
267
+
268
+ # config/initializers/delayed_job.rb
269
+ Delayed::Worker.default_queue_name = 'default'
270
+
271
+ Defaults:
272
+
273
+ delayed_job_command: bin/delayed_job
274
+
275
+
276
+
277
+ ## letsencrypt
278
+
279
+ This creates a single certificate for every server alias/server name in the configuration file.
280
+
281
+ letsencrypt_email: "me@example.com"
282
+ server_name: app.example.com
283
+
284
+
285
+ If you'd like more control over the cert, you can customize the variable `le_ssl_cert` as follows:
286
+
287
+ le_ssl_cert:
288
+ cert_name: "{{server_name}}"
289
+ preferred_challenges: "http"
290
+ plugin: standalone
291
+ domains: "{{ [server_name] + server_aliases }}"
292
+
293
+ For example, to force a manual DNS challenge you can do the following:
294
+
295
+ le_ssl_cert:
296
+ cert_name: star_example
297
+ preferred_challenges: dns
298
+ plugin: manual
299
+ domains:
300
+ - example.com
301
+ - "*.example.com"
302
+
303
+ (you will need to futz around the first time and manually install the DNS record, but it should work on renewals)
304
+
305
+ Note that this role needs to be included _before_ the webserver (apache or nginx) role
306
+
307
+
308
+ ## logrotate
309
+
310
+ Installs logrotate and lets you configure logs for automatic rotation. Example config for rails:
311
+
312
+ logrotate_scripts:
313
+ - name: rails
314
+ path: "/u/apps/{{project_name}}/shared/log/{{rails_env}}.log"
315
+ options:
316
+ - weekly
317
+ - size 100M
318
+ - missingok
319
+ - compress
320
+ - delaycompress
321
+ - copytruncate
322
+
323
+ ## memcache
324
+
325
+ Installs memcache on the server. By default, memcache will only listen on localhost which needs to be changed if other servers needs to connect.
326
+
327
+ # Default Value
328
+ memcache_bind: "127.0.0.1"
329
+
330
+ # bind to all interfaces
331
+ memcache_bind: "0.0.0.0"
332
+
333
+ ## newrelic-infra
334
+ This role will install the next-gen "Newrelic One" infrastructure agent which can perform a few different functions for newrelic. The previous "newrelic" role is deprecated.
335
+
336
+ Variables:
337
+
338
+ # Required, the newrelic license key you get after signing up.
339
+ newrelic_license: "longhashthingyougetfromnewrelichere"
340
+ # Optional - send logs to newrelic one's log aggregator.
341
+ newrelic_logs:
342
+ - name: rails-production
343
+ path: /u/apps/blah/shared/log/production.log
344
+ - name: nginx-error
345
+ path: /var/log/nginx/error.log
346
+
347
+
348
+
349
+ ## nginx-rails
350
+
351
+ Configures nginx to look at localhost:9292 for the socket/backend connection. If you need to do fancy stuff you should simply override this role
352
+
353
+ subspace override nginx-rails
354
+
355
+ defaults are here, we'll probably add more:
356
+
357
+ client_max_body_size: 4G
358
+ ssl_force_redirect: true
359
+ default_server: true
360
+ keepalive_timeout: 10
361
+ extra_nginx_config: ""
362
+
363
+ Optional variables:
364
+
365
+ asset_cors_allow_origin: Set this to set the Access-Control-Allow-Origin for
366
+ everything in /assets.
367
+
368
+ nginx_proxy_read_timeout: Set [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). This is in seconds. You probably only want to change this if using rack-timeout (although I may be wrong). If using rack-timeout, it should be slightly higher than the rack-timeout timeout. I'm doing 5 seconds higher, but that was arbitrarily chosen.
369
+
370
+ ssl_force_redirect: redirect all HTTP traffic to HTTPS on the same host. Defaults to true and only applies if ssl_enabled is also true.
371
+ extra_nginx_config: anything else you want to configure in the main nginx config block, formatted like:
372
+ extra_nginx_config: |
373
+ proxy_http_version 1.1;
374
+ chunked_transfer_encoding off;
375
+ proxy_buffering off;
376
+ proxy_cache off;
377
+
378
+ ## nodejs
379
+
380
+ Used to install different versions of NodeJS. This uses NodeSource's apt repositories. You must define a variable called `nodejs_version` and choose a major version supported by NodeSource:
381
+
382
+ nodejs_version: 14.x
383
+ nodejs_version: 17.x
384
+ nodejs_version: lts
385
+ nodejs_version: current
386
+
387
+ The full list of distributions is here: https://github.com/nodesource/distributions#installation-instructions
388
+
389
+ ## papertrail
390
+
391
+ ## passenger
392
+
393
+ ## postgresql
394
+
395
+ Sets up a postgres *server* - only use this on the database machine.
396
+
397
+ backups_enabled: true
398
+ s3_db_backup_bucket: disabled
399
+ s3_db_backup_prefix: "{{project_name}}/{{rails_env}}"
400
+ database_user: "{{project_name}}"
401
+
402
+ ## puma
403
+ Use the puma app server for your rails app. Usually combined with nginx to server as a static file server and reverse proxy.
404
+
405
+ **Prerequesites:**
406
+ - add `gem puma` to your gemfile
407
+ - add `config/puma/` to the `linked_dirs` config in capistrano's `deploy.rb`
408
+
409
+ This role will generate a reasonable `puma.rb` and configure it to be controlled by systemd.
410
+
411
+ **Variables:**
412
+
413
+ puma_workers: 1 # Puma process count (usually == vCPU count)
414
+ puma_min_threads: 4 # Min threads/process
415
+ puma_max_threads: 16 # Max threads/process
416
+
417
+ puma_max_memory: 1.5G # Use cgroups via systemd to limit memory
418
+ puma_max_cpu: 80% # Use cgroups via systemd to set a CPU quota
419
+
420
+ ## rails
421
+
422
+ Provisions for a rails app. This one is probably pretty important.
423
+
424
+ We no longer provider default values, so make sure to define all the following variables:
425
+
426
+ rails_env: production
427
+ database_pool: 5
428
+ database_name: "{{project_name}}_{{rails_env}}"
429
+ database_user: "{{project_name}}"
430
+ database_host: localhost
431
+ database_adapter: postgresql
432
+ database_password: # usually defined in the encrypted vault
433
+ job_queues:
434
+ - default
435
+ - mailers
436
+
437
+ ## redis
438
+
439
+ Installs redis on the server.
440
+
441
+ # Change to * if you want this available everywhere instead of localhost
442
+ redis_bind: 127.0.0.1
443
+
444
+ As of Subspace 3.0, this uses the official redis apt repo instead of the debian/ubuntu ones. If you previously had installed redis from the distro, you will need to manually uninstall, purge, and reinstall. This should not delete any data but back it up just in case.
445
+
446
+ sudo apt-get purge redis-server
447
+ sudo apt-get install redis-server
448
+
449
+ ## resque
450
+
451
+ Install monitoring and automatic startup for resque workers via monit. You MUST set the `job_queues` variable as follows:
452
+
453
+ job_queues:
454
+ - default
455
+ - mailers
456
+ - exports
457
+
458
+ redis_bind: "*"
459
+
460
+ ## ruby-common
461
+
462
+ Installs ruby on the machine. YOu can set a version by picking off the download url and sha hash from ruby-lang.org
463
+
464
+ ruby_version: ruby-2.4.1
465
+ ruby_checksum: a330e10d5cb5e53b3a0078326c5731888bb55e32c4abfeb27d9e7f8e5d000250
466
+ ruby_download_location: 'https://cache.ruby-lang.org/pub/ruby/2.4/ruby-2.4.1.tar.gz'
467
+ bundler_version: 2.0.1
468
+
469
+
470
+ ## sidekiq
471
+
472
+ This will install a monit script that keeps sidekiq running. We spawn one sidekiq instance that manages as many queues as you need. Varaibles of note:
473
+
474
+ # Process these queues on this server
475
+ job_queues:
476
+ - default
477
+ - mailers
478
+
479
+ # Number of sidekiq *processes* to run
480
+ sidekiq_workers: 1
481
+
482
+ * Note that as of v0.4.13, we now also add a unique job queue for each host with its hostname. This is handy if you need to assign a job to a specific host. In general you should use named queues, but occasionally this is useful and there's no harm in having it there unused.
483
+
484
+ Sidekiq uses redis by default, and rails connects to a redis running on localhost by default. However, this role does not depend on redis since in production it's likely redis will be running elsewhere. If you're provisioning a standalone server, make sure to include the redis role.
485
+
486
+ ## tailscale
487
+
488
+ This installs/updates the Tailscale package, and adds/authenticates servers to the 10FW Tailnet.
489
+
490
+ **Prerequesites:**
491
+ - Create an auth key within tailscale, appropriately tagged & granted ACL access for the given project.
492
+ - Add `tailscale_auth_key` to the secrets vault for the environment you wish to provision.
493
+
494
+ **Recommendations:**
495
+
496
+ At the top of `config/subspace/{env}.yml`, add a separate block to allow the tailscale role to execute before all other roles. When the tailscale package is updated, the ssh connection will be broken, and it's best to get this out of the way first thing during a maintain or provision.
497
+
498
+ **Force Reauthentication:**
499
+
500
+ To force authentication with the tailnet, use the following tag when provisioning:
501
+
502
+ subspace provision {env} --tags=tailscale_reauth`
503
+
504
+ ## Other Internal Roles
505
+
506
+ Since ansible doesn't support versioning of roles, we cloned the role here so that it doesn't change unexpectedly. We expect to update from upstream occasionally, please let us know if we're missing something we should have.
507
+
508
+ You should not include these roles directly in your subspace config files. For example, instead of including `zenoamaro.postgresql`, simply include our `postgresql` role which depens on zenoamaro's role.
509
+
510
+ Thanks to the following repositories for making their roles available:
511
+ * https://github.com/zenoamaro/ansible-postgresql
512
+ * https://github.com/mtpereira/ansible-passenger
513
+
514
+
515
+ # Mitogen
516
+
517
+ In order to dramatically speed up ansible, you can install Mitogen: https://github.com/mitogen-hq/mitogen/blob/master/docs/ansible_detailed.rst
518
+
519
+ pip install -g mitogen
520
+
521
+ Subspace will automatically detect this and update your ansible.cfg file so it is blazing fast.
522
+
29
523
 
30
524
  ## Directory Structure
31
525
 
@@ -43,7 +537,13 @@ Contains the template files that get copied over when `subspace init` is ran.
43
537
 
44
538
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
45
539
 
46
- To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
540
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version:
541
+
542
+ 1. update the version number in `version.rb`
543
+ 2. `gem build subspace.gemspec`
544
+ 3. `gem push subspace-x.y.z.gem`
545
+
546
+ This will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
47
547
 
48
548
  ## Contributing
49
549
 
@@ -52,3 +552,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/tenfor
52
552
  ## License
53
553
 
54
554
  The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
555
+
556
+ # Roles and Variables
data/TODO ADDED
@@ -0,0 +1,34 @@
1
+ TODO:
2
+
3
+ subspace init
4
+ - create ansible.cfg
5
+ - Add stuff from zach
6
+ - generate a password and create a file (add to gitignore)
7
+ - create config/provision
8
+ + entire directory tree
9
+ - generate example production.yml, dev.yml
10
+
11
+
12
+ - create the ansible-vault vars files somehow
13
+
14
+
15
+ -- 8/21
16
+
17
+ 1. remove 10fw.net as the default hostname
18
+ 2. autocreate authorized_keys with ~/.ssh/id_rsa.pub
19
+ 3. add readme or something better than this
20
+ 4. combine dev and prod.yml into the same shared tasks and update the provision task to just pass in the hosts on the command line, i think that's a better more canonical "ansible" thing to do than have different playbooks for dev and prod.
21
+ 5. whats up with the "mtpereira.passenger" role? if our role wants to depend on that one (see postgres) than file, but its confusing in the role list in dev.yml
22
+ 6. create a menu where we ask people what setup they want and they choose each step (apache+passenger) or (nginx+puma) or whatever (or have these in a giant provision.yml and just commented out). similarly they should pick their DB. Heroku calls these things "buildpacks" and we should do a similar thing.
23
+ 7. dont use rake use subspace, eg `subspace provision dev`
24
+ 8. not sure why the rake task generates QA and Staging even when the configs dont exist.
25
+ 9. AWS stuff shouldn't be in there by default, its way too specific to our use case i think
26
+ 10. I can't edit the vault files after doing subspace init. OK I see I have to be in the provision dir. We should do that via subspace commands
27
+ 11. Needs to prompt for ruby version too or pull it from .ruby-version (does it do this or did festbuddy get lucky?)
28
+ 12. Need something like `subspace override <rolename>` that will just make a copy of the role in the provision dir so you can fux0r with it -- also need to think about what happens if we udpate subspace, does it automatically update roles for everyone? Maybe it should always copy them locally and make you run `subspace update roles` to avoid `bundle update` wrecking people's production servers
29
+ 13. `One or more undefined variables: 'letsencrypt_email' is undefined`
30
+ 14. We need to figure out app.yml and the secrets file beacause it sucks, also not everyone uses figaro
31
+ 15. more options for SSL, like should it auto redirect, what hostnames are allowed, etc
32
+ 16. needs an option to set the project name instead of just using the directory
33
+ - ok this is the there, but the readme should force you to edit this file, and there needs to be a ton more options there (unless we go with provision.rb for configuration, which we could do)
34
+
data/UPGRADING.md ADDED
@@ -0,0 +1,10 @@
1
+ # Subspace Upgrade Guide
2
+
3
+
4
+ # 2.x -> 3.0
5
+
6
+ - Run subspace init
7
+ - Someone write subspace upgrade3 please
8
+
9
+ # 2.x -> 2.y
10
+ We strive to follow semver so upgrading from 2.x to 2.y should be safe. However, since use of this tool can affect your production infrastructure, we highly recommend reviewing the [CHANGELOG](CHANGELOG.md) when upgrading.
@@ -0,0 +1,13 @@
1
+ ---
2
+ - hosts: localhost
3
+
4
+ tasks:
5
+ - name: Load environment vars
6
+ include_vars:
7
+ file: "{{vars_file}}"
8
+ - name: create some file
9
+ template:
10
+ src: "{{project_path}}/{{src}}"
11
+ dest: "{{project_path}}/{{dest}}"
12
+ delegate_to: localhost
13
+
@@ -0,0 +1,5 @@
1
+ ---
2
+ - hosts: "{{ maintenance_hosts }}"
3
+ become: true
4
+ roles:
5
+ - nginx-maintenance
@@ -0,0 +1,29 @@
1
+ ---
2
+ - name: Enable syslog communication to sensor
3
+ lineinfile:
4
+ dest: "/etc/rsyslog.d/50-default.conf"
5
+ line: "*.* @{{alienvault_usm_anywhere_sensor_ip}}:514"
6
+ state: present
7
+ become: true
8
+ when: ansible_facts["distribution"] == 'Ubuntu'
9
+ tags:
10
+ - alienvault
11
+
12
+ - name: Enable syslog communication to sensor
13
+ lineinfile:
14
+ dest: "/etc/rsyslog.conf"
15
+ line: "*.* @{{alienvault_usm_anywhere_sensor_ip}}:514"
16
+ state: present
17
+ become: true
18
+ when: ansible_facts["distribution"] == 'Amazon'
19
+ tags:
20
+ - alienvault
21
+
22
+ - name: Restart rsyslog
23
+ service:
24
+ name: rsyslog
25
+ enabled: yes
26
+ state: restarted
27
+ become: true
28
+ tags:
29
+ - alienvault
@@ -1,7 +1,4 @@
1
1
  ---
2
2
  server_aliases: []
3
- template_src_path: project.conf
4
- # TODO Replace with LetsEncrypt.
5
- #ssl_enabled: false
6
- #ssl_server_crt: ./files/{{project_name}}/server.crt
7
- #ssl_intermediate_crt: ./files/{{project_name}}/intermediate.crt
3
+ apache_project_conf: project.conf
4
+ ssl_enabled: false
@@ -1,4 +1,12 @@
1
1
  ---
2
2
  - name: apache restart
3
3
  service: name=apache2 state=restarted
4
- sudo: yes
4
+ become: true
5
+
6
+ - name: stop webserver
7
+ service: name=apache2 state=stopped
8
+ become: true
9
+
10
+ - name: start webserver
11
+ service: name=apache2 state=started
12
+ become: true