ndr_dev_support 7.3.0 → 7.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41d7ba2652d9fa563d40304d74636a120fabce678f2dac3247be956ab448e477
-  data.tar.gz: 52be1d6408975a0777733c5ab9007098ffe2128360b00f13857c591ac051a0db
+  metadata.gz: 5084b3677de82f1613b2a3d95e2290ca79128285da4a004636ee67d23d1546d5
+  data.tar.gz: 13561b93f13f6739c072382faf65968f59a5b82898ccbbb9b3a3c55a006744c3
 SHA512:
-  metadata.gz: d83ee219968cfbffb0f3b2eb9615442c595d6013ba1e0c21f04421197cc68dd94df9338e152922b1e39a566dd4ce7a35543c9672eb6b21548f3f80688b82610a
-  data.tar.gz: fc6eb48554bd132c6bc9bf852ca5d708baaae03c61e36a0dfd8880665d3859418d3ba28a179939a801ba9262c956c41b5db06e5c8f11fd904da7870499c5138f
+  metadata.gz: 656ead5b1726958c845ba2eda14afc1e548e1a012a813aff524dc78953630bae0f119bacdaed63b3621c4da6bfd41439f3022c8ab260a43470408b0f1ef51080
+  data.tar.gz: 541c54385afa983aaa6087302f92318d425556aaf5d9de534c685e52e7dc770e799bc318f0983d83c73053c1d81fe4b7937f14d91143b156292b4387481b5201

data/CHANGELOG.md

@@ -1,6 +1,10 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 7.3.1 / 2025-01-02
+### Added
+* Capistrano: deploy application secrets from a subversion or git repository
+
 ## 7.3.0 / 2024-12-19
 ### Added
 * Capistrano: install rbenv and ruby from /opt/rbenv.tar.gz or vendor/rbenv/

data/lib/ndr_dev_support/capistrano/deploy_secrets.rb

@@ -0,0 +1,166 @@
+# Add a git or svn secrets respository for ndr_dev_support:deploy_secrets
+def add_secrets_repo(name:, url:, scm:, branch: nil)
+  raise "Invalid repo name #{name}" unless /\A[A-Z0-9_-]+\z/i.match?(name)
+  raise "Unknown scm #{scm}" unless %w[svn git].include?(scm)
+  raise "Expected branch for repo #{name}" if scm == 'git' && branch.to_s.empty?
+
+  secrets_repositories = fetch(:secrets_repositories, {})
+  secrets_repositories[name] = { url: url, scm: scm, branch: branch }
+  set :secrets_repositories, secrets_repositories
+end
+
+# Add a secret to be deployed by ndr_dev_support:deploy_secrets
+def add_secret(repo:, repo_path:, shared_dest:)
+  secrets = fetch(:secrets, [])
+  raise "Unknown repo #{repo}" unless fetch(:secrets_repositories, {}).key?(repo)
+
+  secrets << { repo: repo, repo_path: repo_path, shared_dest: shared_dest }
+  set :secrets, secrets
+end
+
+Capistrano::Configuration.instance(:must_exist).load do
+  namespace :ndr_dev_support do
+    desc <<~DESC
+      Deploy updated application secrets to shared folders on application servers
+
+      To use this in a project, add something like the code below to your
+      Capistrano file config/deploy.rb, then run:
+      $ cap target app:update_secrets
+
+      namespace :app do
+        desc 'Update application secrets'
+        task :update_secrets do
+          add_secrets_repo(name: 'userlists',
+                           url: 'https://github.com/example/users.git',
+                           branch: 'main',
+                           scm: 'git')
+          add_secrets_repo(name: 'encrypted_credentials_store',
+                           url: 'https://svn-server.example.org/svn/creds', scm: 'svn')
+
+          add_secret(repo: 'encrypted_credentials_store',
+                     repo_path: 'path/to/credentials.yml.enc',
+                     shared_dest: 'config/credentials.yml.enc')
+          add_secret(repo: 'userlists',
+                     repo_path: 'config/userlist.yml',
+                     shared_dest: 'config/userlist.yml')
+        end
+      end
+      after 'app:update_secrets', 'ndr_dev_support:deploy_secrets'
+    DESC
+    task :deploy_secrets do
+      # List of repositories used for secrets
+      secrets_repositories = fetch(:secrets_repositories, {})
+      secrets = fetch(:secrets, [])
+      secrets_repo_base = Pathname.new('tmp/deployment-secrets')
+
+      if secrets.empty?
+        Capistrano::CLI.ui.say 'Warning: No secret files configured to upload'
+        next
+      end
+
+      # Allow quick indexing by filename
+      secrets_map = secrets.to_h { |secret| [secret[:shared_dest], secret] } # rubocop:disable Rails/IndexBy
+      changed = [] # List of changed files updated
+      Dir.mktmpdir do |secret_dir|
+        # Clone git secrets repositories if required
+        used_repos = secrets.collect { |secret| secret[:repo] }.uniq
+        repo_dirs = {}
+        used_repos.each do |repo|
+          repository = secrets_repositories[repo]
+          next unless repository[:scm] == 'git'
+
+          repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
+          if File.directory?(repo_dir)
+            ok = system("cd #{Shellwords.escape(repo_dir)} && git fetch")
+            raise "Error: cannot fetch secrets repository #{repo}: aborting" unless ok
+          else
+            ok = system('git', 'clone', '--mirror', '--filter=blob:none', repository[:url], repo_dir)
+            raise "Error: cannot clone secrets repository #{repo}: aborting" unless ok
+          end
+          repo_dirs[repo] = repo_dir
+        end
+
+        # Set up a temporary secrets directory of exported secrets,
+        # creating nested structure if necessary
+        secrets_map.each_value do |secret|
+          repo = secret[:repo]
+          repository = secrets_repositories[repo]
+          raise "Unknown repository #{secret[:repo]}" unless repository
+
+          repo_root = repository[:url]
+          raise 'Unknown / unsupported repository' unless repo_root&.start_with?('https://')
+
+          dest_fname = File.join(secret_dir, secret[:shared_dest])
+          dest_dir = File.dirname(dest_fname)
+          FileUtils.mkdir_p(dest_dir)
+          case repository[:scm]
+          when 'git'
+            repo_dir = Pathname.new(secrets_repo_base).join(".git-#{repo}").to_s
+            ok = system("GIT_DIR=#{Shellwords.escape(repo_dir)} git archive --format=tar " \
+                        "#{Shellwords.escape(repository[:branch])} " \
+                        "#{Shellwords.escape(secret[:repo_path])} | " \
+                        "tar x -Ps %#{Shellwords.escape(secret[:repo_path])}%" \
+                        "#{Shellwords.escape(File.join(secret_dir, secret[:shared_dest]))}% " \
+                        "#{Shellwords.escape(secret[:repo_path])}")
+          when 'svn'
+            ok = system('svn', 'export', '--quiet', "#{repo_root}/#{secret[:repo_path]}",
+                        File.join(secret_dir, secret[:shared_dest]))
+            # TODO: use --non-interactive, and then run again interactively if there's an eror
+          else
+            raise "Error: unsupported scm #{repository[:scm]}"
+          end
+
+          raise 'Error: cannot export secrets files: aborting' unless ok
+
+          secret[:digest] = Digest::SHA256.file(dest_fname).hexdigest
+        end
+
+        # Retrieve digests of secrets from application server
+        escaped_fnames = secrets_map.keys.collect { |fname| Shellwords.escape(fname) }
+        capture("cd #{shared_path.shellescape}; " \
+                "sha256sum #{escaped_fnames.join(' ')} || true").split("\n").each do |digest_line|
+          match = digest_line.match(/([0-9a-f]{64}) [ *](.*)/)
+          raise "Invalid digest returned: #{digest_line}" unless match && secrets_map.key?(match[2])
+
+          secrets_map[match[2]][:server_digest] = match[1]
+        end
+
+        # Upload replacements for all changed files
+        secrets_map.each_value do |secret|
+          if secret[:digest] == secret[:server_digest]
+            # Capistrano::CLI.ui.say "Unchanged secret: #{secret[:shared_dest]}"
+            next
+          end
+
+          Capistrano::CLI.ui.say "Uploading changed secret file: #{secret[:shared_dest]}"
+          changed << secret[:shared_dest]
+          # Capistrano does an in-place overwrite of the file, so use a temporary name,
+          # then move it into place
+          temp_dest = capture("mktemp -p #{shared_path.shellescape}").chomp
+          dest_fname = File.join(secret_dir, secret[:shared_dest])
+          put File.read(dest_fname), temp_dest
+          escape_shared_dest = Shellwords.escape(secret[:shared_dest])
+          escape_temp_dest = Shellwords.escape(temp_dest)
+          capture("cd #{shared_path.shellescape}; " \
+                  "chmod 664 #{escape_temp_dest}; " \
+                  "if [ -e #{escape_shared_dest} ]; then cp -p #{escape_shared_dest}{,.orig}; fi; " \
+                  "mv #{escape_temp_dest} #{escape_shared_dest}")
+        end
+      end
+
+      if changed.empty?
+        Capistrano::CLI.ui.say 'No changed secret files to upload'
+      else
+        Capistrano::CLI.ui.say "Uploaded #{changed.size} changed secret files: #{changed.join(', ')}"
+      end
+      # TODO: Support logging of changes, so that a calling script can report changes
+
+      # TODO: maintain a per-target local cache of latest revisions uploaded / file checksums
+      # then we don't need to re-connect to the remote servers, if nothing changed,
+      # We could also then only need to do "svn ls" instead of "svn export"
+
+      # TODO: Warn if some repos are inaccessible?
+      # TODO: Add notes for passwordless SSH deployment, using ssh-agent
+    end
+  end
+end

data/lib/ndr_dev_support/capistrano/ndr_model.rb

@@ -2,6 +2,7 @@ require 'rainbow'
 
 # Discrete bits of functionality we use automatically:
 require_relative 'assets'
+require_relative 'deploy_secrets'
 require_relative 'install_ruby'
 require_relative 'restart'
 require_relative 'revision_logger'

data/lib/ndr_dev_support/version.rb

@@ -2,5 +2,5 @@
 # This defines the NdrDevSupport version. If you change it, rebuild and commit the gem.
 # Use "rake build" to build the gem, see rake -T for all bundler rake tasks (and our own).
 module NdrDevSupport
-  VERSION = '7.3.0'
+  VERSION = '7.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_dev_support
 version: !ruby/object:Gem::Version
-  version: 7.3.0
+  version: 7.3.1
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-19 00:00:00.000000000 Z
+date: 2025-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -409,6 +409,7 @@ files:
 - lib/minitest/rake_ci_plugin.rb
 - lib/ndr_dev_support.rb
 - lib/ndr_dev_support/capistrano/assets.rb
+- lib/ndr_dev_support/capistrano/deploy_secrets.rb
 - lib/ndr_dev_support/capistrano/install_ruby.rb
 - lib/ndr_dev_support/capistrano/ndr_model.rb
 - lib/ndr_dev_support/capistrano/restart.rb