rails_template_18f 0.8.1 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/Gemfile.lock +58 -28
- data/README.md +5 -5
- data/Rakefile +12 -0
- data/SECURITY.md +18 -0
- data/lib/generators/rails_template18f/active_storage/active_storage_generator.rb +2 -9
- data/lib/generators/rails_template18f/active_storage/templates/oscal/component-definitions/active_storage/component-definition.json +69 -0
- data/lib/generators/rails_template18f/circleci/circleci_generator.rb +3 -3
- data/lib/generators/rails_template18f/circleci/templates/bin/ci-server-start +1 -1
- data/lib/generators/rails_template18f/circleci/templates/circleci/config.yml.tt +60 -17
- data/lib/generators/rails_template18f/circleci/templates/docker-compose.ci.yml +1 -0
- data/lib/generators/rails_template18f/circleci/templates/oscal/component-definitions/circleci/component-definition.json.tt +310 -0
- data/lib/generators/rails_template18f/github_actions/github_actions_generator.rb +7 -12
- data/lib/generators/rails_template18f/github_actions/templates/github/actions/setup-languages/action.yml.tt +1 -1
- data/lib/generators/rails_template18f/github_actions/templates/github/actions/setup-project/action.yml.tt +1 -0
- data/lib/generators/rails_template18f/github_actions/templates/github/actions/trestle-cmd/action.yml +20 -0
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/assemble-ssp.yml.tt +26 -0
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/brakeman-analysis.yml +6 -2
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/dependency-scans.yml +4 -4
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/deploy-production.yml.tt +15 -28
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/deploy-staging.yml.tt +15 -28
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/owasp-daily-scan.yml.tt +8 -3
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/owasp-scan.yml.tt +4 -3
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/pa11y.yml.tt +6 -2
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/rspec.yml.tt +1 -1
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/terraform-production.yml +20 -58
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/terraform-staging.yml +20 -58
- data/lib/generators/rails_template18f/github_actions/templates/github/workflows/validate-ssp.yml.tt +46 -0
- data/lib/generators/rails_template18f/github_actions/templates/oscal/component-definitions/github_actions/component-definition.json.tt +310 -0
- data/lib/generators/rails_template18f/i18n/i18n_generator.rb +7 -6
- data/lib/generators/rails_template18f/i18n/templates/config/locales/en.yml.tt +1 -0
- data/lib/generators/rails_template18f/i18n/templates/config/locales/es.yml +2 -0
- data/lib/generators/rails_template18f/i18n/templates/config/locales/fr.yml +2 -0
- data/lib/generators/rails_template18f/i18n/templates/config/locales/zh.yml +2 -0
- data/lib/generators/rails_template18f/newrelic/newrelic_generator.rb +2 -37
- data/lib/generators/rails_template18f/newrelic/templates/oscal/component-definitions/newrelic/component-definition.json.tt +113 -0
- data/lib/generators/rails_template18f/oscal/oscal_generator.rb +55 -17
- data/lib/generators/rails_template18f/oscal/templates/bin/trestle.tt +10 -0
- data/lib/generators/rails_template18f/oscal/templates/doc/compliance/oscal/trestle-config.yaml.tt +6 -0
- data/lib/generators/rails_template18f/sidekiq/sidekiq_generator.rb +2 -2
- data/lib/generators/rails_template18f/terraform/templates/terraform/README.md.tt +37 -52
- data/lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/import.sh +1 -0
- data/lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/main.tf.tt +7 -10
- data/lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/providers.tf +3 -3
- data/lib/generators/rails_template18f/terraform/templates/terraform/bootstrap/run.sh.tt +27 -0
- data/lib/generators/rails_template18f/terraform/templates/terraform/production/main.tf.tt +33 -50
- data/lib/generators/rails_template18f/terraform/templates/terraform/production/providers.tf.tt +8 -2
- data/lib/generators/rails_template18f/terraform/templates/terraform/staging/main.tf.tt +26 -40
- data/lib/generators/rails_template18f/terraform/templates/terraform/staging/providers.tf.tt +8 -2
- data/lib/rails_template18f/generators/base.rb +9 -24
- data/lib/rails_template18f/generators/pipeline_options.rb +0 -172
- data/lib/rails_template18f/version.rb +1 -1
- data/rails-template-18f.gemspec +6 -6
- data/railsrc +1 -0
- data/railsrc-hotwire +1 -0
- data/template.rb +48 -44
- data/templates/Brewfile +2 -2
- data/templates/README.md.tt +3 -4
- data/templates/app/assets/stylesheets/uswds-components.scss +6 -0
- data/templates/app/assets/stylesheets/uswds-overrides/_index.scss +2 -0
- data/templates/app/assets/stylesheets/uswds-overrides/_override-usa-banner.scss +13 -0
- data/templates/app/assets/stylesheets/uswds-overrides/_override-usa-language-selector.scss +38 -0
- data/templates/app/views/application/_header.html.erb +6 -5
- data/templates/app/views/application/_language_selector.html.erb +35 -0
- data/templates/app/views/application/_usa_banner.html.erb +21 -16
- data/templates/bin/ops/create_service_account.sh.tt +11 -5
- data/templates/bin/owasp-scan +3 -3
- data/templates/bin/pa11y-scan +1 -1
- data/templates/config/environments/ci.rb +1 -0
- data/templates/pa11y.js +12 -0
- data/templates/pa11yci.js +8 -0
- metadata +31 -16
- data/templates/pa11yci +0 -9
- /data/lib/generators/rails_template18f/circleci/templates/{Dockerfile.tt → Dockerfile.ci.tt} +0 -0
@@ -24,7 +24,7 @@ module RailsTemplate18f
|
|
24
24
|
|
25
25
|
def install_gem
|
26
26
|
return if gem_installed?("newrelic_rpm")
|
27
|
-
gem "newrelic_rpm", "~>
|
27
|
+
gem "newrelic_rpm", "~> 9.10"
|
28
28
|
bundle_install
|
29
29
|
end
|
30
30
|
|
@@ -58,42 +58,7 @@ EOB
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def update_oscal_doc
|
61
|
-
|
62
|
-
insert_into_oscal "si-4.md", <<~EOS, after: "## Implementation a.\n"
|
63
|
-
New Relic is used for the purposes of monitoring and analyzing #{app_name} application data. New Relic monitors each application within #{app_name} for
|
64
|
-
basic container utilization (CPU, memory, disk) as a baseline of provided metrics. New Relic dashboards are used by #{app_name} operations to obtain
|
65
|
-
near real-time views into the metrics obtained from each application. New Relic provides application metrics that give insight into request/response rates,
|
66
|
-
failure rates, etc. New Relic uses this data to detect anomalies (such as potential unauthorized activity) and alerts the #{app_name} team via <<INSERT NOTIFICATION CHANNEL>>
|
67
|
-
in the GSA Slack. Example: a spike in failed requests may indicate an unauthorized user has entered the system and is attempting to phish for PII.
|
68
|
-
|
69
|
-
1. A subset of relevant specific metrics #{app_name} is constantly monitoring include:
|
70
|
-
* Abnormal cpu, memory, and disk utilization (defined in New Relic alerting rules)
|
71
|
-
* Number of incoming requests
|
72
|
-
* Request response time
|
73
|
-
* Application crashes (for any reason)
|
74
|
-
* Response status codes (high numbers of failing requests would be abnormal)
|
75
|
-
* Applications (by name)
|
76
|
-
* Abnormally high request rates
|
77
|
-
1. Metrics that can be audited within #{app_name} include:
|
78
|
-
* SSH Sessions (disabled in production under normal circumstances)
|
79
|
-
1. A subset of relevant incidents #{app_name} will use these metrics to protect against include:
|
80
|
-
* Unauthorized Access / Intrusion to #{app_name} as an Administrator
|
81
|
-
* Denial of Service (DoS)
|
82
|
-
* Improper Usage
|
83
|
-
* Malicious Code
|
84
|
-
* System Uptime
|
85
|
-
* High Resource Usage
|
86
|
-
|
87
|
-
When suspicious activity is encountered #{app_name} Operations audit the event through the cloud.gov logs provided at logs.fr.cloud.gov
|
88
|
-
(a Kibana instance allowing users to access, filter, and search their cloud.gov logs. These logs are retained automatically by cloud.gov for 180 days after creation.
|
89
|
-
EOS
|
90
|
-
insert_into_oscal "si-4.md", "The #{app_name} application logs events to stdout and stderr which are ingested by cloud.gov and New Relic.", after: "## Implementation c.\n"
|
91
|
-
insert_into_oscal "si-4.md", "#{app_name} Operations are responsible for monitoring the New Relic dashboards that report on specific application events and performing follow-up investigations where necessary.", after: "## Implementation d.\n"
|
92
|
-
insert_into_oscal "si-4.2.md", <<~EOS
|
93
|
-
#{app_name} is monitored using New Relic Application Performance Monitoring (APM),
|
94
|
-
Synthetics and Logs, which detects and alerts on abnormal responses from #{app_name} applications.
|
95
|
-
EOS
|
96
|
-
end
|
61
|
+
copy_oscal_component "newrelic"
|
97
62
|
end
|
98
63
|
|
99
64
|
no_tasks do
|
@@ -0,0 +1,113 @@
|
|
1
|
+
{
|
2
|
+
"component-definition": {
|
3
|
+
"uuid": "7bbcdbff-c3d8-497f-a0fc-3ec96f4acc2d",
|
4
|
+
"metadata": {
|
5
|
+
"title": "New Relic System Monitoring Component Definition.",
|
6
|
+
"last-modified": "2024-06-11T12:51:11.662524+00:00",
|
7
|
+
"version": "0.0.1",
|
8
|
+
"oscal-version": "1.1.2"
|
9
|
+
},
|
10
|
+
"components": [
|
11
|
+
{
|
12
|
+
"uuid": "8eb58925-2761-4de3-86cb-72af189fe378",
|
13
|
+
"type": "service",
|
14
|
+
"title": "New Relic",
|
15
|
+
"description": "New Relic Application Performance Monitoring",
|
16
|
+
"props": [
|
17
|
+
{
|
18
|
+
"name": "Rule_Id",
|
19
|
+
"value": "properly-configured",
|
20
|
+
"remarks": "rule-config"
|
21
|
+
},
|
22
|
+
{
|
23
|
+
"name": "Rule_Description",
|
24
|
+
"value": "System Owner has configured the system to properly utilize New Relic",
|
25
|
+
"remarks": "rule-config"
|
26
|
+
}
|
27
|
+
],
|
28
|
+
"control-implementations": [
|
29
|
+
{
|
30
|
+
"uuid": "7ba2642f-5cfa-431c-a030-afffc4e6a8d4",
|
31
|
+
"source": "trestle://profiles/lato/profile.json",
|
32
|
+
"description": "",
|
33
|
+
"implemented-requirements": [
|
34
|
+
{
|
35
|
+
"uuid": "fae8766e-7bf2-4d77-9c88-db5b2e9a8bfd",
|
36
|
+
"control-id": "si-4",
|
37
|
+
"description": "REPLACE_ME",
|
38
|
+
"props": [
|
39
|
+
{
|
40
|
+
"name": "implementation-status",
|
41
|
+
"value": "planned"
|
42
|
+
}
|
43
|
+
],
|
44
|
+
"statements": [
|
45
|
+
{
|
46
|
+
"statement-id": "si-4_smt.a",
|
47
|
+
"uuid": "850fcb05-724a-46a3-9faf-2574624ef1ee",
|
48
|
+
"description": "New Relic is used for the purposes of monitoring and analyzing <%= app_name %> application data. New Relic monitors each application within <%= app_name %> for basic container utilization (CPU, memory, disk) as a baseline of provided metrics. New Relic dashboards are used by <%= app_name %> operations to obtain near real-time views into the metrics obtained from each application. New Relic provides application metrics that give insight into request/response rates, failure rates, etc. New Relic uses this data to detect anomalies (such as potential unauthorized activity) and alerts the <%= app_name %> team via <<INSERT NOTIFICATION CHANNEL>> in the GSA Slack. Example: a spike in failed requests may indicate an unauthorized user has entered the system and is attempting to phish for PII.\n\n1. A subset of relevant specific metrics <%= app_name %> is constantly monitoring include:\n* Abnormal cpu, memory, and disk utilization (defined in New Relic alerting rules)\n* Number of incoming requests\n* Request response time\n* Application crashes (for any reason)\n* Response status codes (high numbers of failing requests would be abnormal)\n* Applications (by name)\n* Abnormally high request rates\n1. Metrics that can be audited within <%= app_name %> include:\n* SSH Sessions (disabled in production under normal circumstances)\n1. A subset of relevant incidents <%= app_name %> will use these metrics to protect against include:\n* Unauthorized Access / Intrusion to <%= app_name %> as an Administrator\n* Denial of Service (DoS)\n* Improper Usage\n* Malicious Code\n* System Uptime\n* High Resource Usage\n\nWhen suspicious activity is encountered <%= app_name %> Operations audit the event through the cloud.gov logs provided at logs.fr.cloud.gov\n(a Kibana instance allowing users to access, filter, and search their cloud.gov logs. These logs are retained automatically by cloud.gov for 180 days after creation.",
|
49
|
+
"props": [
|
50
|
+
{
|
51
|
+
"name": "Rule_Id",
|
52
|
+
"value": "properly-configured"
|
53
|
+
},
|
54
|
+
{
|
55
|
+
"name": "implementation-status",
|
56
|
+
"value": "implemented"
|
57
|
+
}
|
58
|
+
]
|
59
|
+
},
|
60
|
+
{
|
61
|
+
"statement-id": "si-4_smt.c",
|
62
|
+
"uuid": "dddcc80f-f715-4ee8-acf0-e4d9df3576c5",
|
63
|
+
"description": "The <%= app_name %> application logs events to stdout and stderr which are ingested by cloud.gov and New Relic.",
|
64
|
+
"props": [
|
65
|
+
{
|
66
|
+
"name": "Rule_Id",
|
67
|
+
"value": "properly-configured"
|
68
|
+
},
|
69
|
+
{
|
70
|
+
"name": "implementation-status",
|
71
|
+
"value": "implemented"
|
72
|
+
}
|
73
|
+
]
|
74
|
+
},
|
75
|
+
{
|
76
|
+
"statement-id": "si-4_smt.d",
|
77
|
+
"uuid": "017e8dab-cbd3-4054-9185-cf24d6dcd6b9",
|
78
|
+
"description": "\\<%= app_name %> Operations are responsible for monitoring the New Relic dashboards that report on specific application events and performing follow-up investigations where necessary.",
|
79
|
+
"props": [
|
80
|
+
{
|
81
|
+
"name": "Rule_Id",
|
82
|
+
"value": "properly-configured"
|
83
|
+
},
|
84
|
+
{
|
85
|
+
"name": "implementation-status",
|
86
|
+
"value": "implemented"
|
87
|
+
}
|
88
|
+
]
|
89
|
+
}
|
90
|
+
]
|
91
|
+
},
|
92
|
+
{
|
93
|
+
"uuid": "fc6a4cb3-5160-4792-a835-bfdf92d97a33",
|
94
|
+
"control-id": "si-4.2",
|
95
|
+
"description": "\\<%= app_name %> is monitored using New Relic Application Performance Monitoring (APM), Synthetics and Logs, which detects and alerts on abnormal responses from <%= app_name %> applications in real time.",
|
96
|
+
"props": [
|
97
|
+
{
|
98
|
+
"name": "Rule_Id",
|
99
|
+
"value": "properly-configured"
|
100
|
+
},
|
101
|
+
{
|
102
|
+
"name": "implementation-status",
|
103
|
+
"value": "implemented"
|
104
|
+
}
|
105
|
+
]
|
106
|
+
}
|
107
|
+
]
|
108
|
+
}
|
109
|
+
]
|
110
|
+
}
|
111
|
+
]
|
112
|
+
}
|
113
|
+
}
|
@@ -7,35 +7,41 @@ module RailsTemplate18f
|
|
7
7
|
class OscalGenerator < ::Rails::Generators::Base
|
8
8
|
include Base
|
9
9
|
|
10
|
-
class_option :oscal_repo,
|
11
|
-
class_option :
|
12
|
-
class_option :branch, desc: "Name of the branch to switch to when using a submodule. Defaults to `
|
10
|
+
class_option :oscal_repo, desc: "GitHub Repo to store compliance documents within. Leave blank to check docs into the app repo"
|
11
|
+
class_option :tag, desc: "Which docker-trestle tag to use. Defaults to `latest`"
|
12
|
+
class_option :branch, desc: "Name of the branch to switch to when using a submodule. Defaults to `main`"
|
13
13
|
|
14
14
|
desc <<~DESC
|
15
15
|
Description:
|
16
|
-
|
17
|
-
submodule for documenting security controls.
|
16
|
+
Set up doc/compliance/oscal as a working directory for use with https://github.com/GSA-TTS/docker-trestle.
|
18
17
|
|
19
18
|
This generator is still experimental.
|
20
19
|
|
21
|
-
Prerequisite:
|
20
|
+
Optional Prerequisite:
|
22
21
|
|
23
|
-
|
22
|
+
Set up a separate private repository to store the compliance documentation in if access control needs to be
|
23
|
+
tighter than for the code. This generator will set up the directory as a submodule so developers with access
|
24
|
+
can easily update documentation alongside code. Updates to the documentation
|
24
25
|
will be pushed to this fork, not the rails app repository.
|
25
26
|
DESC
|
26
27
|
|
27
|
-
def
|
28
|
-
if
|
29
|
-
git clone: "#{options[:oscal_repo]} doc/compliance/oscal"
|
30
|
-
remove_dir "doc/compliance/oscal/.git"
|
31
|
-
else
|
28
|
+
def configure_compliance_files
|
29
|
+
if use_submodule?
|
32
30
|
git submodule: "add #{options[:oscal_repo]} doc/compliance/oscal"
|
33
31
|
inside "doc/compliance/oscal" do
|
34
32
|
git switch: "-c #{branch_name}"
|
35
33
|
end
|
34
|
+
else
|
35
|
+
create_file "doc/compliance/oscal/.keep"
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
+
def copy_templates
|
40
|
+
template "bin/trestle"
|
41
|
+
chmod "bin/trestle", 0o755
|
42
|
+
template "doc/compliance/oscal/trestle-config.yaml"
|
43
|
+
end
|
44
|
+
|
39
45
|
def update_readme
|
40
46
|
if file_content("README.md").match?("## Documentation")
|
41
47
|
insert_into_file "README.md", readme_contents, after: "## Documentation\n"
|
@@ -45,7 +51,7 @@ module RailsTemplate18f
|
|
45
51
|
end
|
46
52
|
|
47
53
|
def configure_submodule
|
48
|
-
|
54
|
+
if use_submodule?
|
49
55
|
git config: "-f .gitmodules submodule.\"doc/compliance/oscal\".branch #{branch_name}"
|
50
56
|
git config: "diff.submodule log"
|
51
57
|
git config: "status.submodulesummary 1"
|
@@ -53,9 +59,24 @@ module RailsTemplate18f
|
|
53
59
|
end
|
54
60
|
end
|
55
61
|
|
62
|
+
def configure_gitignore
|
63
|
+
unless skip_git? || use_submodule?
|
64
|
+
append_to_file ".gitignore", <<~EOM
|
65
|
+
|
66
|
+
# Trestle working files
|
67
|
+
doc/compliance/oscal/.trestle/_trash
|
68
|
+
doc/compliance/oscal/.trestle/cache
|
69
|
+
EOM
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
56
73
|
no_tasks do
|
57
74
|
def branch_name
|
58
|
-
options[:branch].present? ? options[:branch] :
|
75
|
+
options[:branch].present? ? options[:branch] : "main"
|
76
|
+
end
|
77
|
+
|
78
|
+
def docker_trestle_tag
|
79
|
+
options[:tag].present? ? options[:tag] : "latest"
|
59
80
|
end
|
60
81
|
|
61
82
|
def readme_contents
|
@@ -64,8 +85,25 @@ module RailsTemplate18f
|
|
64
85
|
### Compliance Documentation
|
65
86
|
|
66
87
|
Security Controls should be documented within doc/compliance/oscal.
|
88
|
+
|
89
|
+
* Run `bin/trestle` to start the trestle CLI.
|
90
|
+
* Run `bin/trestle SCRIPT_NAME` to run a single trestle script
|
91
|
+
|
92
|
+
#### Initial trestle setup.
|
93
|
+
|
94
|
+
These steps must happen once per project.
|
95
|
+
|
96
|
+
1. Docker desktop must be running
|
97
|
+
1. Start the trestle cli with `bin/trestle`
|
98
|
+
1. Copy the `cloud_gov` component to the local workspace with `copy-component -n cloud_gov`
|
99
|
+
1. Generate the initial markdown with `generate-ssp-markdown`
|
100
|
+
|
101
|
+
#### Ongoing use
|
102
|
+
|
103
|
+
See the [docker-trestle README](https://github.com/gsa-tts/docker-trestle) for help with the workflow
|
104
|
+
for using those scripts for editing the SSP.
|
67
105
|
README
|
68
|
-
return content
|
106
|
+
return content unless use_submodule?
|
69
107
|
<<~README
|
70
108
|
#{content}
|
71
109
|
|
@@ -94,8 +132,8 @@ module RailsTemplate18f
|
|
94
132
|
README
|
95
133
|
end
|
96
134
|
|
97
|
-
def
|
98
|
-
options[:
|
135
|
+
def use_submodule?
|
136
|
+
options[:oscal_repo].present?
|
99
137
|
end
|
100
138
|
end
|
101
139
|
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
#! /usr/bin/env bash
|
2
|
+
|
3
|
+
command="bash"
|
4
|
+
if [ "$1" != "" ]; then
|
5
|
+
command=$1
|
6
|
+
fi
|
7
|
+
|
8
|
+
oscal_location="$(dirname "$(realpath "$0")")/../doc/compliance/oscal"
|
9
|
+
|
10
|
+
docker run -it --rm -v $oscal_location:/app/docs:rw ghcr.io/gsa-tts/trestle:<%= docker_trestle_tag %> $command
|
@@ -14,7 +14,7 @@ module RailsTemplate18f
|
|
14
14
|
|
15
15
|
def install_gem
|
16
16
|
return if gem_installed?("sidekiq")
|
17
|
-
gem "sidekiq", "~>
|
17
|
+
gem "sidekiq", "~> 7.2"
|
18
18
|
bundle_install
|
19
19
|
end
|
20
20
|
|
@@ -24,7 +24,7 @@ module RailsTemplate18f
|
|
24
24
|
# queue for sidekiq jobs
|
25
25
|
brew "redis"
|
26
26
|
EOB
|
27
|
-
insert_into_file "README.md", indent("* [redis]()\n"), after: /\* Install homebrew dependencies: `brew bundle`\n/
|
27
|
+
insert_into_file "README.md", indent("* [redis](https://redis.io/)\n"), after: /\* Install homebrew dependencies: `brew bundle`\n/
|
28
28
|
end
|
29
29
|
|
30
30
|
def configure_server_runner
|
@@ -4,26 +4,39 @@ This directory holds the terraform modules for maintaining your complete persist
|
|
4
4
|
|
5
5
|
Prerequisite: install the `jq` JSON processor: `brew bundle` or `brew install jq`
|
6
6
|
|
7
|
-
## Initial setup
|
7
|
+
## Initial project setup
|
8
8
|
|
9
|
-
|
9
|
+
These steps only need to be run once per project.
|
10
|
+
|
11
|
+
1. Manually [bootstrap the state storage bucket](#bootstrapping-the-state-storage-s3-buckets-for-the-first-time) within the `bootstrap` directory
|
10
12
|
1. Setup CI/CD Pipeline to run Terraform
|
11
|
-
|
12
|
-
|
13
|
-
|
13
|
+
1. Copy bootstrap credentials to your CI/CD secrets using the instructions in the base README
|
14
|
+
1. Create a cloud.gov SpaceDeployer by following the instructions under `SpaceDeployers`
|
15
|
+
1. Copy SpaceDeployer credentials to your CI/CD secrets using the instructions in the base README
|
14
16
|
1. Manually Running Terraform
|
15
|
-
|
17
|
+
1. Follow instructions under `Set up a new environment` to create your infrastructure
|
18
|
+
|
19
|
+
## Initial developer setup
|
20
|
+
|
21
|
+
These steps should be run for any developer that needs to start running terraform or who just moved to a new machine.
|
22
|
+
|
23
|
+
They are not necessary for the developer who runs the [initial project setup](#initial-project-setup)
|
24
|
+
|
25
|
+
1. Import the existing bootstrap resources to your local state with `./import.sh`
|
26
|
+
1. Follow instructions under [Use bootstrap credentials](#use-bootstrap-credentials)
|
27
|
+
|
16
28
|
|
17
29
|
## Terraform State Credentials
|
18
30
|
|
19
|
-
The bootstrap module is used to create an s3 bucket for later terraform runs to store their state in.
|
31
|
+
The `bootstrap` module is used to create an s3 bucket for later terraform runs to store their state in.
|
20
32
|
|
21
33
|
### Bootstrapping the state storage s3 buckets for the first time
|
22
34
|
|
23
|
-
|
24
|
-
|
35
|
+
These steps are run once per project.
|
36
|
+
|
37
|
+
1. Run `./run.sh init`
|
25
38
|
1. Run `./run.sh apply` to set up the bucket and retrieve credentials
|
26
|
-
1. Follow instructions under
|
39
|
+
1. Follow instructions under [Use bootstrap credentials](#use-bootstrap-credentials)
|
27
40
|
1. Ensure that `import.sh` includes a line and correct IDs for any resources created
|
28
41
|
1. Run `./teardown_creds.sh` to remove the space deployer account used to create the s3 bucket
|
29
42
|
|
@@ -31,28 +44,20 @@ The bootstrap module is used to create an s3 bucket for later terraform runs to
|
|
31
44
|
|
32
45
|
*This should not be necessary in most cases*
|
33
46
|
|
34
|
-
1. Run `terraform init`
|
35
|
-
1. If you don't have terraform state locally:
|
36
|
-
1. run `./import.sh`
|
37
|
-
1. optionally run `./run.sh apply` to include the existing outputs in the state file
|
38
47
|
1. Make your changes
|
39
|
-
1.
|
40
|
-
|
41
|
-
### Retrieving existing bucket credentials
|
48
|
+
1. Run `./run.sh plan` to verify the changes are what you expect
|
49
|
+
1. Continue from step 2 of the [boostrapping instructions](#bootstrapping-the-state-storage-s3-buckets-for-the-first-time)
|
42
50
|
|
43
|
-
|
44
|
-
1. Follow instructions under `Use bootstrap credentials`
|
45
|
-
|
46
|
-
#### Use bootstrap credentials
|
51
|
+
### Use bootstrap credentials
|
47
52
|
|
48
53
|
1. Add the following to `~/.aws/credentials`
|
49
54
|
```
|
50
55
|
[<%= app_name %>-terraform-backend]
|
51
|
-
aws_access_key_id = <
|
52
|
-
aws_secret_access_key = <
|
56
|
+
aws_access_key_id = <AWS_ACCESS_KEY_ID from run.sh output>
|
57
|
+
aws_secret_access_key = <AWS_SECRET_ACCESS_KEY from run.sh output>
|
53
58
|
```
|
54
59
|
|
55
|
-
1. Copy `
|
60
|
+
1. Copy `BUCKET` from `run.sh` output to the backend block of `staging/providers.tf` and `production/providers.tf`
|
56
61
|
|
57
62
|
## SpaceDeployers
|
58
63
|
|
@@ -63,11 +68,11 @@ deploy the application from the CI/CD pipeline. Create a new account by running:
|
|
63
68
|
|
64
69
|
## Set up a new environment manually
|
65
70
|
|
66
|
-
The below steps rely on you first configuring access to the Terraform state in s3 as described in [
|
71
|
+
The below steps rely on you first configuring access to the Terraform state in s3 as described in [initial project setup](#initial-project-setup) or [initial developer setup](#initial-developer-setup).
|
67
72
|
|
68
73
|
1. `cd` to the environment you are working in
|
69
74
|
|
70
|
-
1. Set up a SpaceDeployer
|
75
|
+
1. Set up a SpaceDeployer and save the credentials in a file named `secrets.auto.tfvars`
|
71
76
|
```bash
|
72
77
|
# create a space deployer service instance that can log in with just a username and password
|
73
78
|
# the value of < SPACE_NAME > should be `staging` or `prod` depending on where you are working
|
@@ -80,17 +85,17 @@ The below steps rely on you first configuring access to the Terraform state in s
|
|
80
85
|
|
81
86
|
The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your `<ACCOUNT_NAME>`. Read more in the [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/).
|
82
87
|
|
83
|
-
The easiest way to use this script is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in
|
88
|
+
The easiest way to use this script locally is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in
|
84
89
|
|
85
90
|
1. Run terraform from your new environment directory with
|
86
91
|
```bash
|
87
|
-
terraform init
|
92
|
+
terraform init -backend-config="profile=<%= app_name %>-terraform-backend"
|
88
93
|
terraform plan
|
89
94
|
```
|
90
95
|
|
91
96
|
1. Apply changes with `terraform apply`.
|
92
97
|
|
93
|
-
1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform
|
98
|
+
1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform plan before letting CI/CD apply the changes.
|
94
99
|
```bash
|
95
100
|
# <SPACE_NAME> and <ACCOUNT_NAME> have the same values as used above.
|
96
101
|
../../bin/ops/destroy_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME>
|
@@ -98,7 +103,7 @@ The below steps rely on you first configuring access to the Terraform state in s
|
|
98
103
|
|
99
104
|
## Structure
|
100
105
|
|
101
|
-
Each environment has its own module
|
106
|
+
Each environment has its own module.
|
102
107
|
|
103
108
|
```
|
104
109
|
- bootstrap/
|
@@ -111,38 +116,18 @@ Each environment has its own module, which relies on a shared module for everyth
|
|
111
116
|
- <env>/
|
112
117
|
|- main.tf
|
113
118
|
|- providers.tf
|
114
|
-
|- secrets.auto.tfvars
|
115
119
|
|- variables.tf
|
116
|
-
- shared/
|
117
|
-
|- s3/
|
118
|
-
|- main.tf
|
119
|
-
|- providers.tf
|
120
|
-
|- variables.tf
|
121
|
-
|- database/
|
122
|
-
|- main.tf
|
123
|
-
|- providers.tf
|
124
|
-
|- variables.tf
|
125
|
-
|- domain/
|
126
|
-
|- main.tf
|
127
|
-
|- providers.tf
|
128
|
-
|- variables.tf
|
129
120
|
```
|
130
121
|
|
131
|
-
In the shared modules:
|
132
|
-
- `providers.tf` contains set up instructions for Terraform about Cloud Foundry and AWS
|
133
|
-
- `main.tf` sets up the data and resources the application relies on
|
134
|
-
- `variables.tf` lists the required variables and applicable default values
|
135
|
-
|
136
122
|
In the environment-specific modules:
|
137
123
|
- `providers.tf` lists the required providers
|
138
124
|
- `main.tf` calls the shared Terraform code, but this is also a place where you can add any other services, resources, etc, which you would like to set up for that environment
|
139
125
|
- `variables.tf` lists the variables that will be needed, either to pass through to the child module or for use in this module
|
140
|
-
- `secrets.auto.tfvars` is a file which contains the information about the service-key and other secrets that should not be shared
|
141
126
|
|
142
127
|
In the bootstrap module:
|
143
128
|
- `providers.tf` lists the required providers
|
144
129
|
- `main.tf` sets up s3 bucket to be shared across all environments. It lives in `prod` to communicate that it should not be deleted
|
145
130
|
- `variables.tf` lists the variables that will be needed. Most values are hard-coded in this module
|
146
|
-
- `run.sh` Helper script to set up a space deployer and run terraform. The terraform action (`show`/`plan`/`apply`/`destroy`) is passed as an argument
|
131
|
+
- `run.sh` Helper script to set up a space deployer and run terraform. The terraform action (`init`/`show`/`plan`/`apply`/`destroy`) is passed as an argument
|
147
132
|
- `teardown_creds.sh` Helper script to remove the space deployer setup as part of `run.sh`
|
148
|
-
- `import.sh` Helper script to create a new local state file
|
133
|
+
- `import.sh` Helper script to create a new local state file when new developers need to access the state file
|
@@ -4,6 +4,7 @@ read -p "Are you sure you want to import terraform state (y/n)? " verify
|
|
4
4
|
|
5
5
|
if [[ $verify == "y" ]]; then
|
6
6
|
echo "Importing bootstrap state"
|
7
|
+
./run.sh init
|
7
8
|
./run.sh import module.s3.cloudfoundry_service_instance.bucket TKTK
|
8
9
|
./run.sh import cloudfoundry_service_key.bucket_creds TKTK
|
9
10
|
./run.sh plan
|
@@ -1,18 +1,14 @@
|
|
1
1
|
locals {
|
2
|
-
cf_api_url = "https://api.fr.cloud.gov"
|
3
2
|
s3_service_name = "<%= app_name %>-terraform-state"
|
4
3
|
}
|
5
4
|
|
6
5
|
module "s3" {
|
7
|
-
source = "github.com/
|
6
|
+
source = "github.com/gsa-tts/terraform-cloudgov//s3?ref=v1.0.0"
|
8
7
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
cf_space_name = "<%= cloud_gov_production_space %>"
|
14
|
-
s3_service_name = local.s3_service_name<% if cloud_gov_organization == "sandbox-gsa" %>
|
15
|
-
s3_plan_name = "basic-sandbox"<% end %>
|
8
|
+
cf_org_name = "<%= cloud_gov_organization %>"
|
9
|
+
cf_space_name = "<%= cloud_gov_production_space %>"
|
10
|
+
name = local.s3_service_name<% if cloud_gov_organization == "sandbox-gsa" %>
|
11
|
+
s3_plan_name = "basic-sandbox"<% end %>
|
16
12
|
}
|
17
13
|
|
18
14
|
resource "cloudfoundry_service_key" "bucket_creds" {
|
@@ -21,5 +17,6 @@ resource "cloudfoundry_service_key" "bucket_creds" {
|
|
21
17
|
}
|
22
18
|
|
23
19
|
output "bucket_credentials" {
|
24
|
-
value
|
20
|
+
value = cloudfoundry_service_key.bucket_creds.credentials
|
21
|
+
sensitive = true
|
25
22
|
}
|
@@ -3,14 +3,14 @@ terraform {
|
|
3
3
|
required_providers {
|
4
4
|
cloudfoundry = {
|
5
5
|
source = "cloudfoundry-community/cloudfoundry"
|
6
|
-
version = "0.
|
6
|
+
version = "0.53.1"
|
7
7
|
}
|
8
8
|
}
|
9
9
|
}
|
10
10
|
|
11
11
|
provider "cloudfoundry" {
|
12
|
-
api_url =
|
12
|
+
api_url = "https://api.fr.cloud.gov"
|
13
13
|
user = var.cf_user
|
14
14
|
password = var.cf_password
|
15
15
|
app_logs_max = 30
|
16
|
-
}
|
16
|
+
}
|
@@ -1,5 +1,20 @@
|
|
1
1
|
#!/usr/bin/env bash
|
2
2
|
|
3
|
+
if ! command -v jq &> /dev/null
|
4
|
+
then
|
5
|
+
echo "jq must be installed. Run 'brew bundle' to install everything in the Brewfile"
|
6
|
+
exit 1
|
7
|
+
fi
|
8
|
+
if ! command -v terraform &> /dev/null
|
9
|
+
then
|
10
|
+
echo "terraform must be installed before running this script"
|
11
|
+
exit 1
|
12
|
+
fi
|
13
|
+
|
14
|
+
dig_output () {
|
15
|
+
dig_result=`cat terraform.tfstate | jq -r ".outputs.bucket_credentials.value.$1"`
|
16
|
+
}
|
17
|
+
|
3
18
|
if [[ ! -f "secrets.auto.tfvars" ]]; then
|
4
19
|
../../bin/ops/create_service_account.sh -s <%= cloud_gov_production_space %> -u config-bootstrap-deployer > secrets.auto.tfvars
|
5
20
|
fi
|
@@ -7,6 +22,18 @@ fi
|
|
7
22
|
if [[ $# -gt 0 ]]; then
|
8
23
|
echo "Running terraform $@"
|
9
24
|
terraform $@
|
25
|
+
if [[ -f terraform.tfstate ]]; then
|
26
|
+
echo
|
27
|
+
echo "Credentials for terraform state bucket:"
|
28
|
+
dig_output "bucket"
|
29
|
+
echo "BUCKET=$dig_result"
|
30
|
+
dig_output "access_key_id"
|
31
|
+
echo "AWS_ACCESS_KEY_ID=$dig_result"
|
32
|
+
dig_output "secret_access_key"
|
33
|
+
echo "AWS_SECRET_ACCESS_KEY=$dig_result"
|
34
|
+
dig_output "region"
|
35
|
+
echo "AWS_REGION=$dig_result"
|
36
|
+
fi
|
10
37
|
else
|
11
38
|
echo "Not running terraform"
|
12
39
|
fi
|