opswalrus 1.0.30 → 1.0.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 312fb5f2541ced41eeec5b3b7eb1cc6774d0884dfc02487675dfe974d10504fa
-  data.tar.gz: 40658c9a4515ab5d5d5652e301991163136a339399c21f2acc37ea3546716e80
+  metadata.gz: c48630a4544c676d6a2dd18c77799907e19c413fcc0fb5e8d4e5437990877c71
+  data.tar.gz: 39f117ef87f69593b58dfccede64b3a0da0b034abc56e78f59a386362ab052c9
 SHA512:
-  metadata.gz: a24a2a3dc3e0287c0ed1cae15284dc45943dd209c18a5c9698cd8632c3a07d5da68e3abd4af468ff79511f42eb40fd17a72e1508ebe9358a492b288a51b0faa3
-  data.tar.gz: b240ef837c47d9b942c3c61c88e3ffbf73db88016374c037e012e8d5c901a1e6d07774591a9d91b14080232c04dc6e4c210cf9334bb8d6adc3988de637c48573
+  metadata.gz: d0966d03fafbacaf5034bc13b785a49cff7a887ee5a2059c29c9db8f2df8a0a9a2e9cfbba2eeae0360088b89dee67a3a1cc7331ae9dedc8e3ed00fc52b818241
+  data.tar.gz: c8fed3b4eb606f7be0703187d9a0f2297ea96c2b937bfba8b30ca5a4f5bd43a5e921aa0b9ef187dc7a8c52bcd86484e65393e6b49e6d3667ab103f9b2e3b352a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    opswalrus (1.0.30)
+    opswalrus (1.0.32)
       amazing_print (~> 1.5)
       bcrypt_pbkdf (~> 1.1)
       citrus (~> 3.0)

data/build.ops

@@ -49,7 +49,7 @@ bw_status_output = bw_status_output.gsub('mac failed.', '').strip
 bw_status_json = bw_status_output.parse_json
 
 if bw_status_json['status'] != 'unlocked'
-  exit 0, "Bitwarden is not unlocked. Please unlock bitwarden with: bw unlock"
+  exit 1, "Bitwarden is not unlocked. Please unlock bitwarden with: bw unlock"
 end
 
 totp = sh("Get Rubygems OTP") { 'bw get totp Rubygems' }

data/lib/opswalrus/app.rb

@@ -1,7 +1,6 @@
 require "citrus"
 require "io/console"
 require "json"
-# require "logger"
 require "random/formatter"
 require "pastel"
 require "pathname"
@@ -10,11 +9,14 @@ require "shellwords"
 require "socket"
 require "stringio"
 require "yaml"
-require_relative "errors"
+
 require_relative "patches"
+
+require_relative "errors"
 require_relative "git"
 require_relative "host"
 require_relative "hosts_file"
+require_relative "inventory"
 require_relative "operation_runner"
 require_relative "bundler"
 require_relative "package_file"
@@ -33,6 +35,7 @@ module OpsWalrus
 
 
     attr_reader :local_hostname
+    attr_reader :identity_file_paths
 
     def initialize(pwd = Dir.pwd)
       SemanticLogger.default_level = :info
@@ -49,6 +52,7 @@ module OpsWalrus
       @verbose = false
       @sudo_user = nil
       @sudo_password = nil
+      @identity_file_paths = []
       @inventory_host_references = []
       @inventory_tag_selections = []
       @params = nil
@@ -93,6 +97,10 @@ module OpsWalrus
       @local_hostname = hostname.empty? ? "localhost" : hostname
     end
 
+    def set_identity_files(*paths)
+      @identity_file_paths = paths.flatten.compact.uniq
+    end
+
     def set_inventory_hosts(*hosts)
       hosts.flatten!.compact!
       @inventory_host_references.concat(hosts).compact!
@@ -381,29 +389,29 @@ module OpsWalrus
       tags = @inventory_tag_selections + (tag_selection || [])
       tags.uniq!
 
-      host_references = ["hosts.yaml"] if (host_references.nil? || host_references.empty?) && File.exist?("hosts.yaml")
+      host_references = [HostsFile::DEFAULT_FILE_NAME] if (host_references.nil? || host_references.empty?) && File.exist?(HostsFile::DEFAULT_FILE_NAME)
 
-      hosts_files, host_strings = host_references.partition {|ref| File.exist?(ref) }
-      hosts_files = hosts_files.map {|file_path| HostsFile.new(file_path) }
-      untagged_hosts = host_strings.map(&:strip).uniq.map {|host| Host.new(host) }
-      inventory_file_hosts = hosts_files.reduce({}) do |host_map, hosts_file|
-        hosts_file.hosts.each do |host|
-          (host_map[host] ||= host).tag!(host.tags)
-        end
+      Inventory.new(host_references, tags).hosts
+    end
 
-        host_map
-      end.keys
-      all_hosts = untagged_hosts + inventory_file_hosts
+    def edit_inventory(file_path)
+      raise "File not found: #{file_path}" unless File.exist?(file_path)
 
-      selected_hosts = if tags.empty?
-        all_hosts
-      else
-        all_hosts.select do |host|
-          tags.all? {|t| host.tags.include? t }
-        end
-      end
+      HostsFile.edit(file_path)
+    end
+
+    def encrypt_inventory(file_path, output_file_path)
+      raise "File not found: #{file_path}" unless File.exist?(file_path)
+
+      hosts_file = HostsFile.new(file_path)
+      hosts_file.encrypt(output_file_path)
+    end
+
+    def decrypt_inventory(file_path, output_file_path)
+      raise "File not found: #{file_path}" unless File.exist?(file_path)
 
-      selected_hosts.sort_by(&:to_s)
+      hosts_file = HostsFile.new(file_path)
+      hosts_file.decrypt(output_file_path)
     end
 
     def print_version

data/lib/opswalrus/cli.rb

@@ -4,6 +4,11 @@ require "gli"
 require_relative "app"
 
 module OpsWalrus
+  def self.env_specified_age_ids()
+    # ENV['AGE_ID'] || (ENV['OPSWALRUS_AGE_IDS'] && Dir.glob(ENV['OPSWALRUS_AGE_IDS']))
+    ENV['OPSWALRUS_AGE_IDS'] && Dir.glob(ENV['OPSWALRUS_AGE_IDS'])
+  end
+
   class Cli
     extend GLI::App
 
@@ -37,6 +42,7 @@ module OpsWalrus
 
     flag [:h, :hosts], multiple: true, desc: "Specify the hosts.yaml file"
     flag [:t, :tags], multiple: true, desc: "Specify a set of tags to filter the hosts by"
+    flag [:i, :id], multiple: true, desc: "Specify one or more Age Encryption identify files (private keys)"
 
     desc 'Print version'
     command :version do |c|
@@ -45,18 +51,75 @@ module OpsWalrus
       end
     end
 
-    desc 'Report on the host inventory'
-    long_desc 'Report on the host inventory'
+    desc 'View and edit host inventory'
+    long_desc 'View and edit host inventory'
     command :inventory do |c|
-      c.action do |global_options, options, args|
-        hosts = global_options[:hosts] || []
-        tags = global_options[:tags] || []
 
-        log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
-        $app.set_log_level(log_level)
+      desc 'List hosts in inventory'
+      long_desc 'List hosts in inventory'
+      c.command [:ls, :list] do |list|
+        list.action do |global_options, options, args|
+
+          hosts = global_options[:hosts]
+          tags = global_options[:tags]
+
+          log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
+          $app.set_log_level(log_level)
+
+          $app.report_inventory(hosts, tags: tags)
+        end
+      end
+
+      desc 'Edit hosts in inventory'
+      long_desc 'Edit the hosts in the inventory and their secrets'
+      # arg_name 'hosts_file', :optional
+      c.command :edit do |edit|
+        edit.action do |global_options, options, args|
+          file_path = global_options[:hosts].first || HostsFile::DEFAULT_FILE_NAME
+
+          id_files = global_options[:id]
+          id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+          $app.set_identity_files(id_files)
+
+          $app.edit_inventory(file_path)
+        end
+      end
+
+      desc 'Encrypt secrets in inventory file'
+      long_desc 'Encrypt secrets in inventory file'
+      arg_name 'encrypted_host_file_path', :optional
+      c.command :encrypt do |encrypt|
+        encrypt.action do |global_options, options, args|
+          file_path = global_options[:hosts].first || HostsFile::DEFAULT_FILE_NAME
+          output_file_path = args.first || file_path
+
+          id_files = global_options[:id]
+          id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+
+          $app.set_identity_files(id_files)
+
+          $app.encrypt_inventory(file_path, output_file_path)
+        end
+      end
+
+      desc 'Decrypt secrets in inventory file'
+      long_desc 'Decrypt secrets in inventory file'
+      arg_name 'decrypted_host_file_path', :optional
+      c.command :decrypt do |decrypt|
+        decrypt.action do |global_options, options, args|
+          file_path = global_options[:hosts].first || HostsFile::DEFAULT_FILE_NAME
+          output_file_path = args.first || file_path
 
-        $app.report_inventory(hosts, tags: tags)
+          id_files = global_options[:id]
+          id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+
+          $app.set_identity_files(id_files)
+
+          $app.decrypt_inventory(file_path, output_file_path)
+        end
       end
+
+      c.default_command :list
     end
 
     desc 'Bootstrap a set of hosts to run opswalrus'
@@ -71,12 +134,17 @@ module OpsWalrus
         log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
         $app.set_log_level(log_level)
 
-        hosts = global_options[:hosts] || []
-        tags = global_options[:tags] || []
+        hosts = global_options[:hosts]
+        tags = global_options[:tags]
 
         $app.set_inventory_hosts(hosts)
         $app.set_inventory_tags(tags)
 
+        id_files = global_options[:id]
+        id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+
+        $app.set_identity_files(id_files)
+
         dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
         $app.dry_run! if dry_run
 
@@ -102,8 +170,8 @@ module OpsWalrus
         log_level = global_options[:debug] && :trace || global_options[:verbose] && :debug || :info
         $app.set_log_level(log_level)
 
-        hosts = global_options[:hosts] || []
-        tags = global_options[:tags] || []
+        hosts = global_options[:hosts]
+        tags = global_options[:tags]
 
         $app.set_inventory_hosts(hosts)
         $app.set_inventory_tags(tags)
@@ -115,6 +183,11 @@ module OpsWalrus
 
         $app.set_sudo_user(user) if user
 
+        id_files = global_options[:id]
+        id_files = OpsWalrus.env_specified_age_ids if id_files.empty?
+
+        $app.set_identity_files(id_files)
+
         dry_run = [:noop, :dryrun, :dry_run].any? {|sym| global_options[sym] || options[sym] }
         $app.dry_run! if dry_run
 

data/lib/opswalrus/host.rb

@@ -1,5 +1,6 @@
 require "set"
 require "sshkit"
+require "tempfile"
 
 require_relative "interaction_handlers"
 require_relative "invocation"
@@ -192,7 +193,7 @@ module OpsWalrus
 
       # cmd = "OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}'; /home/linuxbrew/.linuxbrew/bin/gem exec --conservative -g opswalrus ops"
       # cmd = "OPS_GEM=\"#{OPS_GEM}\" OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}'; $OPS_GEM exec --conservative -g opswalrus ops"
-      cmd = "OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}'; #{OPS_CMD}"
+      cmd = "OPSWALRUS_LOCAL_HOSTNAME='#{local_hostname_for_remote_host}' eval #{OPS_CMD}"
       cmd << " -v" if verbose
       cmd << " #{ops_command.to_s}"
       cmd << " #{ops_command_options.to_s}" if ops_command_options
@@ -207,30 +208,52 @@ module OpsWalrus
   class Host
     include HostDSL
 
-    def initialize(name_or_ip_or_cidr, tags = [], props = {})
+    def initialize(name_or_ip_or_cidr, tags = [], props = {}, default_props = {}, hosts_file)
       @name_or_ip_or_cidr = name_or_ip_or_cidr
       @tags = tags.to_set
       @props = props.is_a?(Array) ? {"tags" => props} : props.to_h
+      @default_props = default_props
+      @hosts_file = hosts_file
+      @tmp_ssh_key_files = []
+    end
+
+    # secret_ref: SecretRef
+    # returns the decrypted value referenced by the supplied SecretRef
+    def dereference_secret_if_needed(secret_ref)
+      if secret_ref.is_a? SecretRef
+        @hosts_file.read_secret(secret_ref.to_s)
+      else
+        secret_ref
+      end
     end
 
     def host
       @name_or_ip_or_cidr
     end
 
+    def alias
+      @props["alias"] || @default_props["alias"]
+    end
+
+    def ignore?
+      @props["ignore"] || @default_props["ignore"]
+    end
+
     def ssh_port
-      @props["port"]
+      @props["port"] || @default_props["port"]
     end
 
     def ssh_user
-      @props["user"]
+      @props["user"] || @default_props["user"]
     end
 
     def ssh_password
-      @props["password"]
+      password = @props["password"] || @default_props["password"]
+      dereference_secret_if_needed(password)
     end
 
-    def ssh_keys
-      @props["keys"]
+    def ssh_key
+      @props["ssh-key"] || @default_props["ssh-key"]
     end
 
     def hash
@@ -245,10 +268,6 @@ module OpsWalrus
       @name_or_ip_or_cidr
     end
 
-    def alias
-      @props["alias"]
-    end
-
     def tag!(*tags)
       enumerables, scalars = tags.partition {|t| Enumerable === t }
       @tags.merge(scalars)
@@ -263,21 +282,52 @@ module OpsWalrus
     def summary(verbose = false)
       report = "#{to_s}\n  tags: #{tags.sort.join(', ')}"
       if verbose
-        @props.reject{|k,v| k == 'tags' }.each {|k,v| report << "\n  #{k}: #{v}" }
+        @default_props.merge(@props).reject{|k,v| k == 'tags' }.each {|k,v| report << "\n  #{k}: #{v}" }
       end
       report
     end
 
     def sshkit_host
+      keys = case ssh_key
+      when Array
+        ssh_key
+      else
+        [ssh_key]
+      end
+      keys = write_temp_ssh_keys_if_needed(keys)
+
+      # the various options for net-ssh are captured in https://net-ssh.github.io/ssh/v1/chapter-2.html
       @sshkit_host ||= ::SSHKit::Host.new({
         hostname: host,
         port: ssh_port || 22,
         user: ssh_user || raise("No ssh user specified to connect to #{host}"),
         password: ssh_password,
-        keys: ssh_keys
+        keys: keys
       })
     end
 
+    # keys is an Array ( String | SecretRef )
+    # such that if a key is a String, then it is interpreted as a path to a key file
+    # and if the key is a SecretRef, then the secret's plaintext value is interpreted
+    # as an ssh key string, and must thereforce be written to a tempfile so that net-ssh
+    # can use it via file reference (since net-ssh only allows the keys field to be an array of file paths).
+    #
+    # returns an array of file paths to key files
+    def write_temp_ssh_keys_if_needed(keys)
+      keys.map do |key_file_path_or_in_memory_key_text|
+        if key_file_path_or_in_memory_key_text.is_a? SecretRef    # we're dealing with an in-memory key file; we need to write it to a tempfile
+          tempfile = Tempfile.new
+          @tmp_ssh_key_files << tempfile
+          key_file_contents = @hosts_file.read_secret(key_file_path_or_in_memory_key_text.to_s)
+          tempfile.write(key_file_contents)
+          tempfile.close(false)   # we want to close the file without unlinking so that the editor can write to it
+          tempfile.path
+        else    # we're dealing with a reference to a keyfile - a path - so return it
+          key_file_path_or_in_memory_key_text
+        end
+      end
+    end
+
     def set_runtime_env(runtime_env)
       @runtime_env = runtime_env
     end
@@ -294,6 +344,8 @@ module OpsWalrus
       @runtime_env = nil
       @sshkit_backend = nil
       @tmp_bundle_root_dir = nil
+      @tmp_ssh_key_files.each {|tmpfile| tmpfile.close() rescue nil; tmpfile.unlink() rescue nil }
+      @tmp_ssh_key_files = []
     end
 
     def execute(*args, input: nil)
@@ -317,6 +369,17 @@ module OpsWalrus
       @sshkit_backend.download!(remote_path.to_s, local_path.to_s)
     end
 
+    def to_h
+      hash = {}
+      hash["alias"] = @props["alias"] if @props["alias"]
+      hash["ignore"] = @props["ignore"] if @props["ignore"]
+      hash["user"] = @props["user"] if @props["user"]
+      hash["port"] = @props["port"] if @props["port"]
+      hash["password"] = @props["password"] if @props["password"]
+      hash["ssh-key"] = @props["ssh-key"] if @props["ssh-key"]
+      hash["tags"] = tags.to_a unless tags.empty?
+      hash
+    end
   end
 
 end

data/lib/opswalrus/hosts_file.rb

@@ -1,16 +1,39 @@
+require "open3"
 require "pathname"
+require "psych"
+require "stringio"
+require "tempfile"
+require "tty-editor"
 require "yaml"
 require_relative "host"
 
 module OpsWalrus
+
   class HostsFile
+    def self.edit(file_path)
+      tempfile = Tempfile.new
+      begin
+        tempfile.close(false)   # we want to close the file without unlinking so that the editor can write to it
+        HostsFile.new(file_path).decrypt(tempfile.path)
+        if TTY::Editor.open(tempfile.path)
+          # tempfile.open()
+          HostsFile.new(tempfile.path).encrypt(file_path)
+        end
+      ensure
+        tempfile.close rescue nil
+        tempfile.unlink   # deletes the temp file
+      end
+    end
+
+    DEFAULT_FILE_NAME = "hosts.yaml"
+
     attr_accessor :hosts_file_path
     attr_accessor :yaml
 
     def initialize(hosts_file_path)
       @hosts_file_path = File.absolute_path(hosts_file_path)
-      @yaml = YAML.load(File.read(hosts_file_path)) if File.exist?(hosts_file_path)
-      # puts @yaml.inspect
+      @yaml = Psych.safe_load(File.read(hosts_file_path), permitted_classes: [SecretRef]) if File.exist?(hosts_file_path)
+      @cipher = AgeEncryptionCipher.new(ids, App.instance.identity_file_paths)
     end
 
     def defaults
@@ -31,14 +54,74 @@ module OpsWalrus
       #   "192.168.56.10"=>{"tags"=>["web", "vagrant"]}
       # }
       @yaml.map do |host_ref, host_attrs|
-        next if host_ref == "default" || host_ref == "defaults"   # this maps to a nil
+        next if ['default', 'defaults', 'secrets', 'ids'].include?(host_ref)
 
         host_params = host_attrs.is_a?(Hash) ? host_attrs : {}
 
-        Host.new(host_ref, tags(host_ref), defaults.merge(host_params))
+        Host.new(host_ref, tags(host_ref), host_params, defaults, self)
       end.compact
     end
 
+    # secrets are key/value pairs in which the key is an identifier used throughout the yaml file to reference the secret's value
+    # and the associated value is either a Hash or a String.
+    # 1. If the secret's value is a Hash, then the Hash must consist of two keys - ids and a secret value:
+    #    - the ids field explicitly names the intended audience for the secret value.
+    #      The value associated with the ids field is either a String value structured as a comma delimited list of ids,
+    #      in which each id is a reference to an id contained within the ids section of the inventory file
+    #      OR
+    #      the ids field is an Array value in which each element of the array is a reference to an id contained within
+    #      the ids section of the inventory file.
+    #    - the value field is a String value storing the secret value
+    # 2. If the secret's value is a String, then the string is the secret value, and is interpreted to be intended
+    #    for use by an audience consisting of all of the ids listed in the ids section of the inventory file.
+    #
+    # returns a Hash of secret-name/Secret pairs
+    def secrets
+      @secrets ||= (@yaml["secrets"] || {}).map do |secret_name, secret_attrs|
+        audience_ids, secret_value = case secret_attrs
+        when Hash
+          id_names = case ids_value = secret_attrs["ids"]
+          when String
+            ids_value.split(',').map(&:strip)
+          when Array
+            ids_value.map {|elem| elem.to_s.strip }
+          else
+            raise "ids field beloning to secret '#{secret_name}' is of an unknown type: #{ids_value.class.name}: #{ids_value.inspect}"
+          end
+          value = secret_attrs["value"]
+          [id_names, value]
+        when String
+          id_names = self.ids.select {|k,id_public_key_or_array_of_id_names| PublicKey === id_public_key_or_array_of_id_names }.keys
+          value = secret_attrs
+          [id_names, value]
+        else
+          raise "Secret '#{secret_name}' has an unexpected type #{secret_attrs.class.name}: #{secret_attrs.inspect}"
+        end
+
+        [secret_name, Secret.new(secret_name, secret_value, audience_ids)]
+      end.to_h
+    end
+
+    # returns a Hash of id-name/(PublicKey | Array String ) pairs
+    def ids
+      @ids ||= begin
+        id_public_key_pairs, alias_id_set_pairs = (@yaml["ids"] || {}).partition{|k,v| String === v }.map(&:to_h)
+
+        named_public_keys = id_public_key_pairs.map do |id_name, public_key_string|
+          [id_name, PublicKey.new(id_name, public_key_string)]
+        end.to_h
+
+        # named_id_sets = alias_id_set_pairs.map do |id_name, id_array|
+        #   referenced_public_keys = id_array.map {|id| named_public_keys[id] }.uniq.compact
+        #   [id_name, referenced_public_keys]
+        # end.to_h
+
+        # named_public_keys.merge(named_id_sets)
+
+        named_public_keys.merge(alias_id_set_pairs)
+      end
+    end
+
     def tags(host)
       host_attrs = @yaml[host]
 
@@ -51,5 +134,273 @@ module OpsWalrus
         tags.compact.uniq
       end || []
     end
+
+    def to_yaml
+      hash = {}
+      hash["defaults"] = defaults unless defaults.empty?
+      hosts.each do |host|
+        hash[host.host] = host.to_h
+      end
+      hash["secrets"] = secrets
+      hash["ids"] = ids
+
+      yaml = Psych.safe_dump(hash, permitted_classes: [SecretRef, Secret, PublicKey])
+      yaml.sub(/^---\s*/,"")    # omit the leading line: ---\n
+    end
+
+    # returns the decrypted value referenced by secret_name
+    def read_secret(secret_name)
+      secret = secrets[secret_name]
+      secret.decrypt(@cipher)
+    end
+
+    def encrypt_secrets!()
+      secrets.each do |secret_name, secret|
+        secret.encrypt(@cipher)
+      end
+    end
+
+    def decrypt_secrets!()
+      secrets.each do |secret_name, secret|
+        secret.decrypt(@cipher)
+      end
+    end
+
+    def decrypt(decrypted_file_path = nil)
+      decrypted_file_path ||= @hosts_file_path
+      puts "Decrypting #{@hosts_file_path} -> #{decrypted_file_path}."
+      raise("Path to age identity not specified") if App.instance.identity_file_paths.empty?
+      decrypt_secrets!
+      File.write(decrypted_file_path, to_yaml)
+      # puts to_yaml
+    end
+
+    def encrypt(encrypted_file_path = nil)
+      encrypted_file_path ||= @hosts_file_path
+      puts "Encrypting #{@hosts_file_path} -> #{encrypted_file_path}."
+      raise("Path to age identity not specified") if App.instance.identity_file_paths.empty?
+      encrypt_secrets!
+      File.write(encrypted_file_path, to_yaml)
+      # puts to_yaml
+    end
   end
+
+  class PublicKey
+    # yaml_tag nil
+
+    def initialize(id_name, public_key)
+      @id = id_name
+      @public_key = public_key
+    end
+
+    def key
+      @public_key
+    end
+
+    # #init_with and #encode_with are demonstrated here:
+    # - https://djellemah.com/blog/2014/07/23/yaml-deserialisation/
+    # - https://stackoverflow.com/questions/10629209/how-can-i-control-which-fields-to-serialize-with-yaml
+    # - https://github.com/protocolbuffers/protobuf/issues/4391
+    # serialise to yaml
+    def encode_with(coder)
+      # per https://rubydoc.info/stdlib/psych/3.0.0/Psych/Coder#scalar-instance_method
+      coder.represent_scalar(nil, @public_key)
+
+      # coder.scalar = @public_key
+      # coder['public_key'] = @public_key
+    end
+
+    # deserialise from yaml
+    def init_with(coder)
+      @public_key = coder.scalar
+      # @public_key = coder['public_key']
+    end
+  end
+
+  class Cipher
+    # id_to_public_key_map is a Hash of id-name/(PublicKey | Array String ) pairs
+    def initialize(id_to_public_key_map)
+      @ids = id_to_public_key_map
+    end
+
+    # returns: PublicKey | nil | Array PublicKey
+    def dereference(audience_id_reference)
+      case ref = @ids[audience_id_reference]
+      when PublicKey
+        ref
+      when Array
+        ref.map {|audience_id_reference| dereference(audience_id_reference) }.flatten.compact.uniq
+      when Nil
+        puts "ID #{audience_id_reference} does not appear in the list of known public key identifiers"
+        nil
+      else
+        raise "ID reference #{audience_id_reference} corresponds to an unknown type of public key or transitive ID reference: #{ref.inspect}"
+      end
+    end
+
+    # returns: Array PublicKey
+    def dereference_all(audience_id_references)
+      audience_id_references.map {|audience_id_reference| dereference(audience_id_reference) }.flatten.compact.uniq
+    end
+
+    # value is the string value to be encrypted
+    # audience_id_references is an Array(String) representing the names associated with public keys in the ids section of the file
+    # returns the encrypted text as a String
+    def encrypt(value, audience_id_references)
+      raise "Not implemented"
+    end
+
+    # returns the decrypted text as a String
+    def decrypt(value)
+      raise "Not implemented"
+    end
+
+    def encrypted?(value)
+      raise "Not implemented"
+    end
+  end
+
+  class AgeEncryption
+    AGE_ENCRYPTED_FILE_HEADER = '-----BEGIN AGE ENCRYPTED FILE-----'
+
+    def self.encrypt(value, public_keys)
+      recipient_args = public_keys.map {|public_key| "-r #{public_key}" }
+      cmd = "age -e -a #{recipient_args.join(' ')}"
+      stdout, stderr, status = Open3.capture3(cmd, stdin_data: value)
+      raise "Failed to run age encryption: `#{cmd}`" unless status.success?
+      stdout
+    end
+
+    def self.decrypt(value, private_key_file_paths)
+      raise "Unable to decrypt the requested value because there is no age encryption identity (private key) specified" if private_key_file_paths.empty?
+      identity_file_args = private_key_file_paths.map {|private_key_file_path| "-i #{private_key_file_path}" }
+      cmd = "age -d  #{identity_file_args.join(' ')}"
+      stdout, stderr, status = Open3.capture3(cmd, stdin_data: value)
+      raise "Failed to run age encryption: `#{cmd}`" unless status.success?
+      stdout
+    end
+  end
+
+  class AgeEncryptionCipher < Cipher
+    # id_to_public_key_map is a Hash of id-name/(PublicKey | Array String ) pairs
+    def initialize(id_to_public_key_map, private_key_file_paths)
+      super(id_to_public_key_map)
+      @private_key_file_paths = private_key_file_paths
+    end
+
+    # value is the string value to be encrypted
+    # audience_id_references is an Array(String) representing the names associated with public keys in the ids section of the file
+    # returns the encrypted text as a String
+    def encrypt(value, audience_id_references)
+      public_keys = dereference_all(audience_id_references)
+      AgeEncryption.encrypt(value, public_keys.map(&:key))
+    end
+
+    # returns the decrypted text as a String
+    def decrypt(value)
+      AgeEncryption.decrypt(value, @private_key_file_paths)
+    end
+
+    def encrypted?(value)
+      value.strip.start_with?(AgeEncryption::AGE_ENCRYPTED_FILE_HEADER)
+    end
+  end
+
+
+  class SecretRef
+    def initialize(secret_name)
+      @secret_name = secret_name
+    end
+
+    # #init_with and #encode_with are demonstrated here:
+    # - https://djellemah.com/blog/2014/07/23/yaml-deserialisation/
+    # - https://stackoverflow.com/questions/10629209/how-can-i-control-which-fields-to-serialize-with-yaml
+    # - https://github.com/protocolbuffers/protobuf/issues/4391
+    # serialise to yaml
+    def encode_with(coder)
+      # The following line seems to have the effect of quoting the @secret_name
+      # coder.style = Psych::Nodes::Mapping::FLOW
+
+      if @secret_name
+        # don't set tag explicitly, let Psych figure it out from yaml_tag
+        # coder.represent_scalar '!days', days.first
+        coder.scalar = @secret_name
+      # else
+      #   # don't set tag explicitly, let Psych figure it out from yaml_tag
+      #   # coder.represent_seq '!days', days.to_a
+        # coder.seq = @secret_name
+      end
+    end
+
+    # deserialise from yaml
+    def init_with(coder)
+      case coder.type
+      when :scalar
+        @secret_name = coder.scalar
+      # when :seq
+      #   @secret_name = coder.seq
+      else
+        raise "Dunno how to handle #{coder.type} for #{coder.inspect}"
+      end
+    end
+
+    def to_s
+      @secret_name
+    end
+  end
+  # YAML.add_domain_type("", "secret") do |type, value|
+  #   SecretRef.new(value)
+  # end
+  YAML.add_tag("!secret", SecretRef)
+
+  class Secret
+    def initialize(name, secret_value, id_references)
+      @name = name
+      @value = secret_value
+      @ids = id_references
+    end
+
+    # #init_with and #encode_with are demonstrated here:
+    # - https://djellemah.com/blog/2014/07/23/yaml-deserialisation/
+    # - https://stackoverflow.com/questions/10629209/how-can-i-control-which-fields-to-serialize-with-yaml
+    # - https://github.com/protocolbuffers/protobuf/issues/4391
+    # serialise to yaml
+    def encode_with(coder)
+      coder.tag = nil
+
+      # per https://rubydoc.info/stdlib/psych/3.0.0/Psych/Coder#scalar-instance_method
+      # coder.represent_scalar(nil, @public_key)
+
+      # coder.scalar = @public_key
+      # puts @ids.inspect
+      single_line_ids = @ids.join(", ")
+      if single_line_ids.size <= 80
+        coder['ids'] = single_line_ids
+      else
+        coder['ids'] = @ids
+      end
+      coder['value'] = @value
+    end
+
+    # deserialise from yaml
+    def init_with(coder)
+      @public_key = coder.scalar
+      # @public_key = coder['public_key']
+    end
+
+    def encrypt(cipher)
+      @value = cipher.encrypt(@value, @ids) unless cipher.encrypted?(@value)
+      @value
+    end
+
+    def decrypt(cipher)
+      @value = cipher.decrypt(@value) if cipher.encrypted?(@value)
+      @value
+    end
+
+    def to_s
+      @value
+    end
+  end
+
 end

data/lib/opswalrus/inventory.rb

@@ -0,0 +1,41 @@
+require "pathname"
+require "tty-editor"
+require "yaml"
+
+module OpsWalrus
+  class Inventory
+    # host_references is an array of host names and  path strings that reference hosts.yaml files
+    # tags is an array of strings
+    def initialize(host_references = [HostsFile::DEFAULT_FILE_NAME], tags = [])
+      @host_references = host_references
+      @tags = tags
+    end
+
+    def hosts()
+      hosts_files, host_strings = @host_references.partition {|ref| File.exist?(ref) }
+      inventory_file_hosts = hosts_files.
+                                map {|file_path| HostsFile.new(file_path) }.
+                                reduce({}) do |host_map, hosts_file|
+                                  hosts_file.hosts.each do |host|
+                                    (host_map[host] ||= host).tag!(host.tags)
+                                  end
+
+                                  host_map
+                                end.
+                                keys
+      untagged_hosts = host_strings.map(&:strip).uniq.map {|host| Host.new(host) }
+      all_hosts = untagged_hosts + inventory_file_hosts
+
+      selected_hosts = if @tags.empty?
+        all_hosts
+      else
+        all_hosts.select do |host|
+          @tags.all? {|t| host.tags.include? t }
+        end
+      end.reject{|host| host.ignore? }
+
+      selected_hosts.sort_by(&:to_s)
+    end
+
+  end
+end

data/lib/opswalrus/version.rb

@@ -1,3 +1,3 @@
 module OpsWalrus
-  VERSION = "1.0.30"
+  VERSION = "1.0.32"
 end

data/vms/web-arch/Vagrantfile

@@ -74,13 +74,19 @@ Vagrant.configure("2") do |config|
   # Enable provisioning with a shell script. Additional provisioners such as
   # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the
   # documentation for more information about their specific syntax and use.
-  config.vm.provision "shell", inline: <<-SHELL
-    sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config
-    sudo systemctl restart sshd
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_ops.pub").first.strip
+    s.inline = <<-SHELL
+      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
 
-    # change vagrant user to require sudo password
-    sudo sed -i 's/vagrant ALL=(ALL) NOPASSWD: ALL/vagrant ALL=(ALL:ALL) ALL/g' /etc/sudoers.d/vagrant
+      sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config
+      sudo systemctl restart sshd
 
-    # sudo sh -c 'echo root:foo | chpasswd'
-  SHELL
+      # change vagrant user to require sudo password
+      sudo sed -i 's/vagrant ALL=(ALL) NOPASSWD: ALL/vagrant ALL=(ALL:ALL) ALL/g' /etc/sudoers.d/vagrant
+
+      # sudo sh -c 'echo root:foo | chpasswd'
+    SHELL
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opswalrus
 version: !ruby/object:Gem::Version
-  version: 1.0.30
+  version: 1.0.32
 platform: ruby
 authors:
 - David Ellis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-26 00:00:00.000000000 Z
+date: 2023-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -194,6 +194,7 @@ files:
 - lib/opswalrus/host.rb
 - lib/opswalrus/hosts_file.rb
 - lib/opswalrus/interaction_handlers.rb
+- lib/opswalrus/inventory.rb
 - lib/opswalrus/invocation.rb
 - lib/opswalrus/local_non_blocking_backend.rb
 - lib/opswalrus/local_pty_backend.rb