ndr_dev_support 7.3.0 → 7.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41d7ba2652d9fa563d40304d74636a120fabce678f2dac3247be956ab448e477
-  data.tar.gz: 52be1d6408975a0777733c5ab9007098ffe2128360b00f13857c591ac051a0db
+  metadata.gz: bb40a54108485d9f25a6cc88323fb474f6fd1b7ba7af787939dc8270bf7ceaf2
+  data.tar.gz: 8120053e06f2443829e917fda979987e9b09e8c568e75781876b338dbfa53ebb
 SHA512:
-  metadata.gz: d83ee219968cfbffb0f3b2eb9615442c595d6013ba1e0c21f04421197cc68dd94df9338e152922b1e39a566dd4ce7a35543c9672eb6b21548f3f80688b82610a
-  data.tar.gz: fc6eb48554bd132c6bc9bf852ca5d708baaae03c61e36a0dfd8880665d3859418d3ba28a179939a801ba9262c956c41b5db06e5c8f11fd904da7870499c5138f
+  metadata.gz: 8685d5ff16b7c3ba001490bc5897647266bef21cfa94b80eaff77678953052afa9b04c4cbedd2fa5ae7652f7fcd5d08ef0f15f12107527608a5d5c601cfdcb9c
+  data.tar.gz: 7c9c95bb049feccf7516c49bc4e2240d43e5aadc8596cbda484163e5fa25f8e4902fd8176c38b9f1be2bee1b3776307701eb369fa0b4b0f0dfdeb189fb1cf0d5

data/CHANGELOG.md

@@ -1,6 +1,21 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 7.3.2 / 2025-05-01
+### Fixed
+* Capistrano: Add missing `tmpdir` requirement to deploy application secrets
+* Capistrano: cap deploy:setup should be safe on existing deployments
+
+### Added
+* Capistrano: add task deploy:preinstall to preinstall ruby and bundled gems
+
+## Changed
+* Drop support for Rails 6.1
+
+## 7.3.1 / 2025-01-02
+### Added
+* Capistrano: deploy application secrets from a subversion or git repository
+
 ## 7.3.0 / 2024-12-19
 ### Added
 * Capistrano: install rbenv and ruby from /opt/rbenv.tar.gz or vendor/rbenv/

data/config/rubocop/ndr.yml

@@ -4,7 +4,7 @@
 #
 # See the README for instructions on using in a project.
 
-require:
+plugins:
   - rubocop-rails
   - rubocop-rake
 

data/lib/ndr_dev_support/capistrano/deploy_secrets.rb

@@ -0,0 +1,168 @@
+require 'tmpdir'
+
+# Add a git or svn secrets respository for ndr_dev_support:deploy_secrets
+def add_secrets_repo(name:, url:, scm:, branch: nil)
+  raise "Invalid repo name #{name}" unless /\A[A-Z0-9_-]+\z/i.match?(name)
+  raise "Unknown scm #{scm}" unless %w[svn git].include?(scm)
+  raise "Expected branch for repo #{name}" if scm == 'git' && branch.to_s.empty?
+
+  secrets_repositories = fetch(:secrets_repositories, {})
+  secrets_repositories[name] = { url: url, scm: scm, branch: branch }
+  set :secrets_repositories, secrets_repositories
+end
+
+# Add a secret to be deployed by ndr_dev_support:deploy_secrets
+def add_secret(repo:, repo_path:, shared_dest:)
+  secrets = fetch(:secrets, [])
+  raise "Unknown repo #{repo}" unless fetch(:secrets_repositories, {}).key?(repo)
+
+  secrets << { repo: repo, repo_path: repo_path, shared_dest: shared_dest }
+  set :secrets, secrets
+end
+
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :ndr_dev_support do
+    desc <<~DESC
+      Deploy updated application secrets to shared folders on application servers
+
+      To use this in a project, add something like the code below to your
+      Capistrano file config/deploy.rb, then run:
+      $ cap target app:update_secrets
+
+      namespace :app do
+        desc 'Update application secrets'
+        task :update_secrets do
+          add_secrets_repo(name: 'userlists',
+                           url: 'https://github.com/example/users.git',
+                           branch: 'main',
+                           scm: 'git')
+          add_secrets_repo(name: 'encrypted_credentials_store',
+                           url: 'https://svn-server.example.org/svn/creds', scm: 'svn')
+
+          add_secret(repo: 'encrypted_credentials_store',
+                     repo_path: 'path/to/credentials.yml.enc',
+                     shared_dest: 'config/credentials.yml.enc')
+          add_secret(repo: 'userlists',
+                     repo_path: 'config/userlist.yml',
+                     shared_dest: 'config/userlist.yml')
+        end
+      end
+      after 'app:update_secrets', 'ndr_dev_support:deploy_secrets'
+    DESC
+    task :deploy_secrets do
+      # List of repositories used for secrets
+      secrets_repositories = fetch(:secrets_repositories, {})
+      secrets = fetch(:secrets, [])
+      secrets_repo_base = Pathname.new('tmp/deployment-secrets')
+
+      if secrets.empty?
+        Capistrano::CLI.ui.say 'Warning: No secret files configured to upload'
+        next
+      end
+
+      # Allow quick indexing by filename
+      secrets_map = secrets.to_h { |secret| [secret[:shared_dest], secret] } # rubocop:disable Rails/IndexBy
+      changed = [] # List of changed files updated
+      Dir.mktmpdir do |secret_dir|
+        # Clone git secrets repositories if required
+        used_repos = secrets.collect { |secret| secret[:repo] }.uniq
+        repo_dirs = {}
+        used_repos.each do |repo|
+          repository = secrets_repositories[repo]
+          next unless repository[:scm] == 'git'
+
+          repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
+          if File.directory?(repo_dir)
+            ok = system("cd #{Shellwords.escape(repo_dir)} && git fetch")
+            raise "Error: cannot fetch secrets repository #{repo}: aborting" unless ok
+          else
+            ok = system('git', 'clone', '--mirror', '--filter=blob:none', repository[:url], repo_dir)
+            raise "Error: cannot clone secrets repository #{repo}: aborting" unless ok
+          end
+          repo_dirs[repo] = repo_dir
+        end
+
+        # Set up a temporary secrets directory of exported secrets,
+        # creating nested structure if necessary
+        secrets_map.each_value do |secret|
+          repo = secret[:repo]
+          repository = secrets_repositories[repo]
+          raise "Unknown repository #{secret[:repo]}" unless repository
+
+          repo_root = repository[:url]
+          raise 'Unknown / unsupported repository' unless repo_root&.start_with?('https://')
+
+          dest_fname = File.join(secret_dir, secret[:shared_dest])
+          dest_dir = File.dirname(dest_fname)
+          FileUtils.mkdir_p(dest_dir)
+          case repository[:scm]
+          when 'git'
+            repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
+            ok = system("GIT_DIR=#{Shellwords.escape(repo_dir)} git archive --format=tar " \
+                        "#{Shellwords.escape(repository[:branch])} " \
+                        "#{Shellwords.escape(secret[:repo_path])} | " \
+                        "tar x -Ps %#{Shellwords.escape(secret[:repo_path])}%" \
+                        "#{Shellwords.escape(File.join(secret_dir, secret[:shared_dest]))}% " \
+                        "#{Shellwords.escape(secret[:repo_path])}")
+          when 'svn'
+            ok = system('svn', 'export', '--quiet', "#{repo_root}/#{secret[:repo_path]}",
+                        File.join(secret_dir, secret[:shared_dest]))
+            # TODO: use --non-interactive, and then run again interactively if there's an eror
+          else
+            raise "Error: unsupported scm #{repository[:scm]}"
+          end
+
+          raise 'Error: cannot export secrets files: aborting' unless ok
+
+          secret[:digest] = Digest::SHA256.file(dest_fname).hexdigest
+        end
+
+        # Retrieve digests of secrets from application server
+        escaped_fnames = secrets_map.keys.collect { |fname| Shellwords.escape(fname) }
+        capture("cd #{shared_path.shellescape}; " \
+                "sha256sum #{escaped_fnames.join(' ')} || true").split("\n").each do |digest_line|
+          match = digest_line.match(/([0-9a-f]{64}) [ *](.*)/)
+          raise "Invalid digest returned: #{digest_line}" unless match && secrets_map.key?(match[2])
+
+          secrets_map[match[2]][:server_digest] = match[1]
+        end
+
+        # Upload replacements for all changed files
+        secrets_map.each_value do |secret|
+          if secret[:digest] == secret[:server_digest]
+            # Capistrano::CLI.ui.say "Unchanged secret: #{secret[:shared_dest]}"
+            next
+          end
+
+          Capistrano::CLI.ui.say "Uploading changed secret file: #{secret[:shared_dest]}"
+          changed << secret[:shared_dest]
+          # Capistrano does an in-place overwrite of the file, so use a temporary name,
+          # then move it into place
+          temp_dest = capture("mktemp -p #{shared_path.shellescape}").chomp
+          dest_fname = File.join(secret_dir, secret[:shared_dest])
+          put File.read(dest_fname), temp_dest
+          escape_shared_dest = Shellwords.escape(secret[:shared_dest])
+          escape_temp_dest = Shellwords.escape(temp_dest)
+          capture("cd #{shared_path.shellescape}; " \
+                  "chmod 664 #{escape_temp_dest}; " \
+                  "if [ -e #{escape_shared_dest} ]; then cp -p #{escape_shared_dest}{,.orig}; fi; " \
+                  "mv #{escape_temp_dest} #{escape_shared_dest}")
+        end
+      end
+
+      if changed.empty?
+        Capistrano::CLI.ui.say 'No changed secret files to upload'
+      else
+        Capistrano::CLI.ui.say "Uploaded #{changed.size} changed secret files: #{changed.join(', ')}"
+      end
+      # TODO: Support logging of changes, so that a calling script can report changes
+
+      # TODO: maintain a per-target local cache of latest revisions uploaded / file checksums
+      # then we don't need to re-connect to the remote servers, if nothing changed,
+      # We could also then only need to do "svn ls" instead of "svn export"
+
+      # TODO: Warn if some repos are inaccessible?
+      # TODO: Add notes for passwordless SSH deployment, using ssh-agent
+    end
+  end
+end

data/lib/ndr_dev_support/capistrano/ndr_model.rb

@@ -2,7 +2,9 @@ require 'rainbow'
 
 # Discrete bits of functionality we use automatically:
 require_relative 'assets'
+require_relative 'deploy_secrets'
 require_relative 'install_ruby'
+require_relative 'preinstall'
 require_relative 'restart'
 require_relative 'revision_logger'
 require_relative 'ruby_version'
@@ -52,11 +54,15 @@ Capistrano::Configuration.instance(:must_exist).load do
       # sticky; all deployments made within it should be owned by the deployer group too. This
       # means that e.g. a deployment by "bob.smith" can then be rolled back by "tom.jones".
       run "mkdir -p #{deploy_to}"
-      run "chgrp -R deployer #{deploy_to}"
+      # Set deployer group for everything created by this user
+      # run "chgrp -R deployer #{deploy_to}"
+      run "find #{deploy_to} -group #{fetch(:user)} -print0 |xargs -r0 chgrp -h deployer"
 
       # The sticky group will apply automatically to new subdirectories, but
       # any existing subdirectories will need it manually applying via `-R`.
-      run "chmod -R g+s #{deploy_to}"
+      # run "chmod -R g+s #{deploy_to}"
+      run "find #{deploy_to} -user #{fetch(:user)} -type d " \
+          '-not -perm -2000 -print0 |xargs -r0 chmod g+s'
     end
 
     desc 'Custom tasks to be run once, after the initial `cap setup`'
@@ -66,8 +72,12 @@ Capistrano::Configuration.instance(:must_exist).load do
         run "mkdir -p #{full_path}"
 
         # Allow the application to write into here:
-        run "chgrp -R #{application_group} #{full_path}"
-        run "chmod -R g+s #{full_path}"
+        # run "chgrp -R #{application_group} #{full_path}"
+        # run "chmod -R g+s #{full_path}"
+        run "find #{full_path} -user #{fetch(:user)} -not -group #{application_group} " \
+            "-print0 |xargs -r0 chgrp -h #{application_group}"
+        run "find #{full_path} -user #{fetch(:user)} -type d " \
+            '-not -perm -2000 -print0 |xargs -r0 chmod g+s'
       end
 
       fetch(:shared_paths, []).each do |path|
@@ -184,6 +194,14 @@ def target_ruby_version_for(env)
   match ? match[:version] : raise('Unrecognized Ruby version!')
 end
 
+def log_deployment_message(msg)
+  name = fetch(:deployer_name, capture('id -un').chomp)
+  log  = File.join(shared_path, 'revisions.log')
+  msg  = "[#{Time.now}] #{name} #{msg}" # rubocop:disable Rails/TimeZone
+
+  run "(test -e #{log} || (touch #{log} && chmod 664 #{log})) && echo #{Shellwords.escape(msg)} >> #{log};"
+end
+
 def add_target(env, name, app, port, app_user, is_web_server)
   desc "Deploy to #{env} service #{app_user || 'you'}@#{app}:#{port}"
   task(name) do

data/lib/ndr_dev_support/capistrano/preinstall.rb

@@ -0,0 +1,37 @@
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :deploy do
+    desc <<~DESC
+      Preinstall ruby and gems, then abort and rollback cleanly, leaving the
+      current installation unchanged.
+
+      This is particularly useful for ruby version bumps: installing the new
+      ruby version and all the bundled gems can take a long time.
+
+      This aborts before updating out-of-bundle gems, in case that causes
+      issues when restarting the currently installed version.
+
+      Usage:
+        cap target deploy:preinstall
+    DESC
+    task :preinstall do
+      # Running this task sets a flag, to make ndr_dev_support:check_preinstall abort.
+      # We do this in a roundabout way on Capistrano 2, because deploy:update_code
+      # explicitly runs deploy:finalize_update, instead of using task dependencies.
+      set :preinstall, true
+    end
+  end
+
+  namespace :ndr_dev_support do
+    desc 'Hook to abort capistrano installation early after preinstalling ruby and in-bundle gems'
+    task :check_preinstall do
+      next unless fetch(:preinstall, false)
+
+      log_deployment_message("preinstalled #{real_revision}")
+      warn Rainbow("Successful preinstall for target: #{fetch(:name)}")
+      abort 'Aborting after successful preinstall'
+    end
+  end
+
+  after 'deploy:preinstall', 'deploy:update'
+  before 'ndr_dev_support:update_out_of_bundle_gems', 'ndr_dev_support:check_preinstall'
+end

data/lib/ndr_dev_support/capistrano/revision_logger.rb

@@ -2,11 +2,7 @@ Capistrano::Configuration.instance(:must_exist).load do
   namespace :ndr_dev_support do
     desc 'Append to the log of deployments the user and revision.'
     task :log_deployment, except: { no_release: true } do
-      name = fetch(:deployer_name, capture('id -un'))
-      log  = File.join(shared_path, 'revisions.log')
-      msg  = "[#{Time.now}] #{name} deployed #{latest_revision}"
-
-      run "(test -e #{log} || (touch #{log} && chmod 664 #{log})) && echo #{msg} >> #{log};"
+      log_deployment_message("deployed #{latest_revision}")
     end
   end
 

data/lib/ndr_dev_support/version.rb

@@ -2,5 +2,5 @@
 # This defines the NdrDevSupport version. If you change it, rebuild and commit the gem.
 # Use "rake build" to build the gem, see rake -T for all bundler rake tasks (and our own).
 module NdrDevSupport
-  VERSION = '7.3.0'
+  VERSION = '7.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_dev_support
 version: !ruby/object:Gem::Version
-  version: 7.3.0
+  version: 7.3.2
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-19 00:00:00.000000000 Z
+date: 2025-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -409,8 +409,10 @@ files:
 - lib/minitest/rake_ci_plugin.rb
 - lib/ndr_dev_support.rb
 - lib/ndr_dev_support/capistrano/assets.rb
+- lib/ndr_dev_support/capistrano/deploy_secrets.rb
 - lib/ndr_dev_support/capistrano/install_ruby.rb
 - lib/ndr_dev_support/capistrano/ndr_model.rb
+- lib/ndr_dev_support/capistrano/preinstall.rb
 - lib/ndr_dev_support/capistrano/restart.rb
 - lib/ndr_dev_support/capistrano/revision_logger.rb
 - lib/ndr_dev_support/capistrano/ruby_version.rb
@@ -485,7 +487,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: NDR Developer Support library