ndr_dev_support 7.2.6 → 7.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 76f81b66bcf17c0e5e413814393d16bc99dd60dd8da6cde93ce57def7cb35e6a
4
- data.tar.gz: ceb801fe0ccbd025342f6a50d0ae7c079be834624339744862a9b6ce010a61a5
3
+ metadata.gz: 5084b3677de82f1613b2a3d95e2290ca79128285da4a004636ee67d23d1546d5
4
+ data.tar.gz: 13561b93f13f6739c072382faf65968f59a5b82898ccbbb9b3a3c55a006744c3
5
5
  SHA512:
6
- metadata.gz: 27ca7a80b76b603d216e4d816aa73f4bfce98cb5fa148502c89df109bea215ed1ad97f94770405b049f2bbf057139cd32901c9019d71891faba069fe298254db
7
- data.tar.gz: a174a2fb75a30a8a716425ae6a9e7892a73827f8f2993df87bb33d4b5226651f38f5fba0cb14101db3aa5b603ed61c40fa8cdfca080b3cea7c30e35ea63ce022
6
+ metadata.gz: 656ead5b1726958c845ba2eda14afc1e548e1a012a813aff524dc78953630bae0f119bacdaed63b3621c4da6bfd41439f3022c8ab260a43470408b0f1ef51080
7
+ data.tar.gz: 541c54385afa983aaa6087302f92318d425556aaf5d9de534c685e52e7dc770e799bc318f0983d83c73053c1d81fe4b7937f14d91143b156292b4387481b5201
data/CHANGELOG.md CHANGED
@@ -1,6 +1,14 @@
1
1
  ## [Unreleased]
2
2
  *no unreleased changes*
3
3
 
4
+ ## 7.3.1 / 2025-01-02
5
+ ### Added
6
+ * Capistrano: deploy application secrets from a subversion or git repository
7
+
8
+ ## 7.3.0 / 2024-12-19
9
+ ### Added
10
+ * Capistrano: install rbenv and ruby from /opt/rbenv.tar.gz or vendor/rbenv/
11
+
4
12
  ## 7.2.6 / 2024-11-13
5
13
  ### Fixed
6
14
  * Support Rails 7.2, 8.0, Ruby 3.3
@@ -4,8 +4,8 @@ Capistrano::Configuration.instance(:must_exist).load do
4
4
  task :configure_assets do
5
5
  asset_script = fetch(:asset_script, <<~SHELL)
6
6
  set -e
7
- ruby -ryaml -e "puts YAML.dump('production' => { 'secret_key_base' => 'compile_me' })" > config/secrets.yml
8
- ruby -ryaml -e "puts YAML.dump('production' => { 'adapter' => 'placeholder' })" > config/database.yml
7
+ ruby -e "require 'yaml'; puts YAML.dump('production' => { 'secret_key_base' => 'compile_me' })" > config/secrets.yml
8
+ ruby -e "require 'yaml'; puts YAML.dump('production' => { 'adapter' => 'placeholder' })" > config/database.yml
9
9
  RAILS_ENV=production bundle exec rake assets:clobber assets:precompile
10
10
  rm config/secrets.yml config/database.yml
11
11
  SHELL
@@ -0,0 +1,166 @@
1
+ # Add a git or svn secrets respository for ndr_dev_support:deploy_secrets
2
+ def add_secrets_repo(name:, url:, scm:, branch: nil)
3
+ raise "Invalid repo name #{name}" unless /\A[A-Z0-9_-]+\z/i.match?(name)
4
+ raise "Unknown scm #{scm}" unless %w[svn git].include?(scm)
5
+ raise "Expected branch for repo #{name}" if scm == 'git' && branch.to_s.empty?
6
+
7
+ secrets_repositories = fetch(:secrets_repositories, {})
8
+ secrets_repositories[name] = { url: url, scm: scm, branch: branch }
9
+ set :secrets_repositories, secrets_repositories
10
+ end
11
+
12
+ # Add a secret to be deployed by ndr_dev_support:deploy_secrets
13
+ def add_secret(repo:, repo_path:, shared_dest:)
14
+ secrets = fetch(:secrets, [])
15
+ raise "Unknown repo #{repo}" unless fetch(:secrets_repositories, {}).key?(repo)
16
+
17
+ secrets << { repo: repo, repo_path: repo_path, shared_dest: shared_dest }
18
+ set :secrets, secrets
19
+ end
20
+
21
+ Capistrano::Configuration.instance(:must_exist).load do
22
+ namespace :ndr_dev_support do
23
+ desc <<~DESC
24
+ Deploy updated application secrets to shared folders on application servers
25
+
26
+ To use this in a project, add something like the code below to your
27
+ Capistrano file config/deploy.rb, then run:
28
+ $ cap target app:update_secrets
29
+
30
+ namespace :app do
31
+ desc 'Update application secrets'
32
+ task :update_secrets do
33
+ add_secrets_repo(name: 'userlists',
34
+ url: 'https://github.com/example/users.git',
35
+ branch: 'main',
36
+ scm: 'git')
37
+ add_secrets_repo(name: 'encrypted_credentials_store',
38
+ url: 'https://svn-server.example.org/svn/creds', scm: 'svn')
39
+
40
+ add_secret(repo: 'encrypted_credentials_store',
41
+ repo_path: 'path/to/credentials.yml.enc',
42
+ shared_dest: 'config/credentials.yml.enc')
43
+ add_secret(repo: 'userlists',
44
+ repo_path: 'config/userlist.yml',
45
+ shared_dest: 'config/userlist.yml')
46
+ end
47
+ end
48
+ after 'app:update_secrets', 'ndr_dev_support:deploy_secrets'
49
+ DESC
50
+ task :deploy_secrets do
51
+ # List of repositories used for secrets
52
+ secrets_repositories = fetch(:secrets_repositories, {})
53
+ secrets = fetch(:secrets, [])
54
+ secrets_repo_base = Pathname.new('tmp/deployment-secrets')
55
+
56
+ if secrets.empty?
57
+ Capistrano::CLI.ui.say 'Warning: No secret files configured to upload'
58
+ next
59
+ end
60
+
61
+ # Allow quick indexing by filename
62
+ secrets_map = secrets.to_h { |secret| [secret[:shared_dest], secret] } # rubocop:disable Rails/IndexBy
63
+ changed = [] # List of changed files updated
64
+ Dir.mktmpdir do |secret_dir|
65
+ # Clone git secrets repositories if required
66
+ used_repos = secrets.collect { |secret| secret[:repo] }.uniq
67
+ repo_dirs = {}
68
+ used_repos.each do |repo|
69
+ repository = secrets_repositories[repo]
70
+ next unless repository[:scm] == 'git'
71
+
72
+ repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
73
+ if File.directory?(repo_dir)
74
+ ok = system("cd #{Shellwords.escape(repo_dir)} && git fetch")
75
+ raise "Error: cannot fetch secrets repository #{repo}: aborting" unless ok
76
+ else
77
+ ok = system('git', 'clone', '--mirror', '--filter=blob:none', repository[:url], repo_dir)
78
+ raise "Error: cannot clone secrets repository #{repo}: aborting" unless ok
79
+ end
80
+ repo_dirs[repo] = repo_dir
81
+ end
82
+
83
+ # Set up a temporary secrets directory of exported secrets,
84
+ # creating nested structure if necessary
85
+ secrets_map.each_value do |secret|
86
+ repo = secret[:repo]
87
+ repository = secrets_repositories[repo]
88
+ raise "Unknown repository #{secret[:repo]}" unless repository
89
+
90
+ repo_root = repository[:url]
91
+ raise 'Unknown / unsupported repository' unless repo_root&.start_with?('https://')
92
+
93
+ dest_fname = File.join(secret_dir, secret[:shared_dest])
94
+ dest_dir = File.dirname(dest_fname)
95
+ FileUtils.mkdir_p(dest_dir)
96
+ case repository[:scm]
97
+ when 'git'
98
+ repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
99
+ ok = system("GIT_DIR=#{Shellwords.escape(repo_dir)} git archive --format=tar " \
100
+ "#{Shellwords.escape(repository[:branch])} " \
101
+ "#{Shellwords.escape(secret[:repo_path])} | " \
102
+ "tar x -Ps %#{Shellwords.escape(secret[:repo_path])}%" \
103
+ "#{Shellwords.escape(File.join(secret_dir, secret[:shared_dest]))}% " \
104
+ "#{Shellwords.escape(secret[:repo_path])}")
105
+ when 'svn'
106
+ ok = system('svn', 'export', '--quiet', "#{repo_root}/#{secret[:repo_path]}",
107
+ File.join(secret_dir, secret[:shared_dest]))
108
+ # TODO: use --non-interactive, and then run again interactively if there's an eror
109
+ else
110
+ raise "Error: unsupported scm #{repository[:scm]}"
111
+ end
112
+
113
+ raise 'Error: cannot export secrets files: aborting' unless ok
114
+
115
+ secret[:digest] = Digest::SHA256.file(dest_fname).hexdigest
116
+ end
117
+
118
+ # Retrieve digests of secrets from application server
119
+ escaped_fnames = secrets_map.keys.collect { |fname| Shellwords.escape(fname) }
120
+ capture("cd #{shared_path.shellescape}; " \
121
+ "sha256sum #{escaped_fnames.join(' ')} || true").split("\n").each do |digest_line|
122
+ match = digest_line.match(/([0-9a-f]{64}) [ *](.*)/)
123
+ raise "Invalid digest returned: #{digest_line}" unless match && secrets_map.key?(match[2])
124
+
125
+ secrets_map[match[2]][:server_digest] = match[1]
126
+ end
127
+
128
+ # Upload replacements for all changed files
129
+ secrets_map.each_value do |secret|
130
+ if secret[:digest] == secret[:server_digest]
131
+ # Capistrano::CLI.ui.say "Unchanged secret: #{secret[:shared_dest]}"
132
+ next
133
+ end
134
+
135
+ Capistrano::CLI.ui.say "Uploading changed secret file: #{secret[:shared_dest]}"
136
+ changed << secret[:shared_dest]
137
+ # Capistrano does an in-place overwrite of the file, so use a temporary name,
138
+ # then move it into place
139
+ temp_dest = capture("mktemp -p #{shared_path.shellescape}").chomp
140
+ dest_fname = File.join(secret_dir, secret[:shared_dest])
141
+ put File.read(dest_fname), temp_dest
142
+ escape_shared_dest = Shellwords.escape(secret[:shared_dest])
143
+ escape_temp_dest = Shellwords.escape(temp_dest)
144
+ capture("cd #{shared_path.shellescape}; " \
145
+ "chmod 664 #{escape_temp_dest}; " \
146
+ "if [ -e #{escape_shared_dest} ]; then cp -p #{escape_shared_dest}{,.orig}; fi; " \
147
+ "mv #{escape_temp_dest} #{escape_shared_dest}")
148
+ end
149
+ end
150
+
151
+ if changed.empty?
152
+ Capistrano::CLI.ui.say 'No changed secret files to upload'
153
+ else
154
+ Capistrano::CLI.ui.say "Uploaded #{changed.size} changed secret files: #{changed.join(', ')}"
155
+ end
156
+ # TODO: Support logging of changes, so that a calling script can report changes
157
+
158
+ # TODO: maintain a per-target local cache of latest revisions uploaded / file checksums
159
+ # then we don't need to re-connect to the remote servers, if nothing changed,
160
+ # We could also then only need to do "svn ls" instead of "svn export"
161
+
162
+ # TODO: Warn if some repos are inaccessible?
163
+ # TODO: Add notes for passwordless SSH deployment, using ssh-agent
164
+ end
165
+ end
166
+ end
@@ -0,0 +1,103 @@
1
+ Capistrano::Configuration.instance(:must_exist).load do
2
+ namespace :ndr_dev_support do
3
+ desc <<~DESC
4
+ Ensure that the required ruby version is installed.
5
+
6
+ This can be installed from /opt/rbenv.tar.gz (first installation only) or vendor/rbenv/
7
+
8
+ To place an offline copy of rbenv in /opt/rbenv.tar.gz
9
+ For ruby 3.1.6, run the following commands:
10
+ $ mkdir clone_rbenv
11
+ $ git clone https://github.com/rbenv/rbenv.git clone_rbenv/.rbenv
12
+ $ git clone https://github.com/rbenv/ruby-build.git clone_rbenv/.rbenv/plugins/ruby-build
13
+ $ mkdir clone_rbenv/.rbenv/cache
14
+ $ (cd clone_rbenv/.rbenv/cache; curl -O https://cache.ruby-lang.org/pub/ruby/3.1/ruby-3.1.6.tar.gz)
15
+ $ (cd clone_rbenv; rm -f ../rbenv.tar.gz; tar czf ../rbenv.tar.gz .rbenv)
16
+ $ rm -rf clone_rbenv
17
+ $ scp -p rbenv.tar.gz app-server:/opt/rbenv.tar.gz
18
+
19
+ To add rbenv, ruby-build and additional ruby versions to the application vendor directory
20
+ For ruby 3.2.6:
21
+ $ mkdir clone_rbenv
22
+ $ git clone https://github.com/rbenv/rbenv.git clone_rbenv/.rbenv
23
+ $ mkdir -p vendor/rbenv; rm -f vendor/rbenv/rbenv.tar.gz
24
+ $ tar czf vendor/rbenv/rbenv.tar.gz -C clone_rbenv .rbenv
25
+ $ rm -rf clone_rbenv
26
+ $ mkdir clone_ruby-build
27
+ $ git clone https://github.com/rbenv/ruby-build.git clone_ruby-build/ruby-build
28
+ $ mkdir -p vendor/rbenv/cache; rm -f vendor/rbenv/ruby-build.tar.gz
29
+ $ tar czf vendor/rbenv/ruby-build.tar.gz -C clone_ruby-build ruby-build
30
+ $ rm -rf clone_ruby-build
31
+ $ (cd vendor/rbenv/cache; curl -O https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.6.tar.gz)
32
+ $ git add vendor/rbenv/{rbenv,ruby-build}.tar.gz vendor/rbenv/cache/*
33
+ DESC
34
+ task :install_ruby do
35
+ version = fetch(:ruby)
36
+ # Note that ruby 3.1.x on CentOS 7 generally installs successfully but then reports an error:
37
+ # ERROR: While executing gem ... (URI::InvalidURIError)
38
+ # bad URI(is not URI?): "bundler\r"
39
+ # For this reason, we ignore the exit status, and run our own test.
40
+ #
41
+ # For some reason, SSH keepalive options have no effect with capistrano 2 and net-ssh 7.
42
+ # So we run a poor-man's keepalive, because this installation can take over 10 minutes
43
+ # and some of our SSH servers disconnect inactive sessions after 5 minutes.
44
+ # We have a keepalive time limit of 40 minutes in case the installation fails unexpectedly.
45
+ #
46
+ # We remove ~/.rbenv paths from capistrano-defined PATH when running `rbenv init`
47
+ # so that it knows the path is needed in ~/.bash_profile
48
+ #
49
+ # We use latest_release: this is broadly well-defined, and will point to the in-progress
50
+ # release if we're part way through a deployment, or the most recent release if run
51
+ # after a deployment has happened, or be blank if attempted after cap deploy:setup
52
+ run <<~SHELL
53
+ set -e;
54
+ if ! rbenv versions --bare 2> /dev/null | grep -q ^#{Regexp.escape(version)}$; then
55
+ echo Installing ruby #{version};
56
+ { sleep 10; for i in `seq 1 80`; do echo -n '.'; sleep 30; done & } 2> /dev/null;
57
+ sudo -i -n -u #{fetch(:application_user)} sh -c "
58
+ if [ ! -e .rbenv ] && [ -e /opt/rbenv.tar.gz ]; then
59
+ tar xf /opt/rbenv.tar.gz .rbenv;
60
+ PATH=\\`echo \\"\\$PATH\\"|sed -e \\"s_[^:=]*/[.]rbenv[^:]*:__g\\"\\` .rbenv/bin/rbenv init bash;
61
+ fi;
62
+ if [ ! -e .rbenv ] && [ -f #{latest_release}/vendor/rbenv/rbenv.tar.gz ]; then
63
+ tar xf #{latest_release}/vendor/rbenv/rbenv.tar.gz .rbenv;
64
+ PATH=\\`echo \\"\\$PATH\\"|sed -e \\"s_[^:=]*/[.]rbenv[^:]*:__g\\"\\` .rbenv/bin/rbenv init bash;
65
+ fi;
66
+ if [ ! -e .rbenv ]; then
67
+ echo rbenv not installed: aborting;
68
+ else
69
+ if [ -d #{latest_release}/vendor/rbenv/cache ] && [ -n \\"\\`ls #{latest_release}/vendor/rbenv/cache\\`\\" ]; then
70
+ mkdir -p .rbenv/cache/;
71
+ cp -nvp #{latest_release}/vendor/rbenv/cache/* .rbenv/cache/;
72
+ fi;
73
+ if [ -f #{latest_release}/vendor/rbenv/ruby-build.tar.gz ]; then
74
+ mkdir -p .rbenv/plugins/;
75
+ tar xf #{latest_release}/vendor/rbenv/ruby-build.tar.gz -C .rbenv/plugins/ ruby-build;
76
+ fi;
77
+ eval \\"\\$(.rbenv/bin/rbenv init - --no-rehash bash)\\";
78
+ export TMPDIR=\\`mktemp -d \\"\\$HOME\\"/rbenv_tmp_XXXX\\`;
79
+ if rbenv install #{version} --skip-existing 2>&1; then
80
+ RBENV_VERSION=#{version} ruby --version;
81
+ rm -rf \\"\\`printenv TMPDIR\\`\\";
82
+ rbenv global #{version};
83
+ fi;
84
+ fi;
85
+ ";
86
+ { kill % && wait; } 2> /dev/null;
87
+ set -e;
88
+ if [ "`RBENV_VERSION=#{version} gem list --exact --installed bundler`" == "true" ]; then
89
+ echo 'Please ignore the following error above:';
90
+ echo '> ERROR: While executing gem ... (URI::InvalidURIError)';
91
+ echo '> bad URI(is not URI?): "bundler\\r"';
92
+ echo Successfully installed ruby #{version};
93
+ else
94
+ echo ERROR: Failure installing ruby #{version}: aborting;
95
+ exit 1;
96
+ fi;
97
+ fi
98
+ SHELL
99
+ end
100
+ end
101
+
102
+ before 'bundle:install', 'ndr_dev_support:install_ruby'
103
+ end
@@ -2,6 +2,8 @@ require 'rainbow'
2
2
 
3
3
  # Discrete bits of functionality we use automatically:
4
4
  require_relative 'assets'
5
+ require_relative 'deploy_secrets'
6
+ require_relative 'install_ruby'
5
7
  require_relative 'restart'
6
8
  require_relative 'revision_logger'
7
9
  require_relative 'ruby_version'
@@ -172,7 +174,7 @@ Capistrano::Configuration.instance(:must_exist).load do
172
174
  end
173
175
 
174
176
  def release_config_for(env)
175
- branches = YAML.load_file('config/deployments.yml')
177
+ branches = YAML.safe_load_file('config/deployments.yml', permitted_classes: [Date])
176
178
  branches.fetch(env.to_s) { raise 'Unknown release branch!' }
177
179
  end
178
180
 
@@ -2,5 +2,5 @@
2
2
  # This defines the NdrDevSupport version. If you change it, rebuild and commit the gem.
3
3
  # Use "rake build" to build the gem, see rake -T for all bundler rake tasks (and our own).
4
4
  module NdrDevSupport
5
- VERSION = '7.2.6'
5
+ VERSION = '7.3.1'
6
6
  end
@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
25
25
  spec.add_dependency 'pry'
26
26
 
27
27
  # Audit dependencies:
28
+ spec.add_dependency 'csv'
28
29
  spec.add_dependency 'highline', '>= 1.6.0'
29
30
 
30
31
  # Rubocop dependencies:
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ndr_dev_support
3
3
  version: !ruby/object:Gem::Version
4
- version: 7.2.6
4
+ version: 7.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - NCRS Development Team
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-11-13 00:00:00.000000000 Z
11
+ date: 2025-01-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -24,6 +24,20 @@ dependencies:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: csv
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: highline
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -395,6 +409,8 @@ files:
395
409
  - lib/minitest/rake_ci_plugin.rb
396
410
  - lib/ndr_dev_support.rb
397
411
  - lib/ndr_dev_support/capistrano/assets.rb
412
+ - lib/ndr_dev_support/capistrano/deploy_secrets.rb
413
+ - lib/ndr_dev_support/capistrano/install_ruby.rb
398
414
  - lib/ndr_dev_support/capistrano/ndr_model.rb
399
415
  - lib/ndr_dev_support/capistrano/restart.rb
400
416
  - lib/ndr_dev_support/capistrano/revision_logger.rb