hybrid_platforms_conductor 33.2.4 → 33.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd578efdc520f51dc9752ace39626e73dbdda736fd4376ee5e9ce55ed34a0a4d
-  data.tar.gz: c44c935905b7a62a4a0f3ea037b0f1e4ed8fa07bdad4802e38143fe1a448bfd1
+  metadata.gz: c0707386d30d5e671d5ceac5e1fec9d971cdccd2b7057c6ed3a75cb5d8bb6d85
+  data.tar.gz: e9e5c023662e7aa9283905f4ae54f4b8995d14f61bea1d844ef4fef32cba68dc
 SHA512:
-  metadata.gz: f0070cf6736acaa5b9a1e36279bab3b9f6530c26ea341feec4fb4775d022e1c67e2694df832a3ab340b765e10bf510969febe82e4b34fa564e9aab82eea71314
-  data.tar.gz: d5cc0229b32b7f75d1c7604e304e6986b0b7ab121676dfd62121d954c48e3ba93f827dabfbb8320c9f945339c334263ff9404c8c60e1be5a0b03e4b390cc044f
+  metadata.gz: 1b70dedf6605d48081ead37771b8f94e29d26334fb1c0f7f15ed8ed70cabc24809b145bd681af9e829cd0c968accd18cf87886987391709a7252908a00104096
+  data.tar.gz: cbe9033432aae4b83b4153501efad2330651696ba0fc8b487d21f5927430d481661e5d3bf2c3617ffb20c2ba7e254f82328d4225aeba4f98295873c55fe4cca9

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# [v33.3.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.2.4...v33.3.0) (2021-07-02 17:20:58)
+
+## Global changes
+### Patches
+
+* [[Feature(secrets_reader_keepass)] [#79] Add Keepass secrets reader plugin to get secrets from KeePass databases](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/7ace48653a74f03ed535ee6b41b21242a6454ff3)
+
+## Changes for secrets_reader_keepass
+### Features
+
+* [[Feature(secrets_reader_keepass)] [#79] Add Keepass secrets reader plugin to get secrets from KeePass databases](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/7ace48653a74f03ed535ee6b41b21242a6454ff3)
+
 # [v33.2.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.2.3...v33.2.4) (2021-06-23 15:14:20)
 
 ## Global changes

data/docs/plugins.md

@@ -196,6 +196,7 @@ Check the [sample plugin file](../lib/hybrid_platforms_conductor/hpc_plugins/sec
 
 Plugins shipped by default:
 * [`cli`](plugins/secrets_reader/cli.md)
+* [`keepass`](plugins/secrets_reader/keepass.md)
 * [`thycotic`](plugins/secrets_reader/thycotic.md)
 
 <a name="test"></a>

data/docs/plugins/secrets_reader/keepass.md

@@ -0,0 +1,62 @@
+# Secrets reader plugin: `keepass`
+
+The `keepass` secrets reader plugin retrieves secrets from [KeePass](https://keepass.info/) databases, using an actual KeePass installation with the [KPScript plugin](https://keepass.info/plugins.html#kpscript).
+
+It is configured by giving the KPScript command-line (using `use_kpscript_from` config DSL method), the KeePass databases to be read (using `secrets_from_keepass` config DSL method) and uses the `keepass` credential ID to authenticate along with extra environment variables for eventual key files or encrypted passwords.
+
+## Config DSL extension
+
+### `use_kpscript_from`
+
+Provide the KPScript command-line to be used. If KPScript is already in your path, using `KPScript.exe` or `kpscript` should be enough, otherwise the full path to the command-line will be needed. On Windows it is needed to also include double quotes if the path contains spaces (like `"C:\Program Files\KeePass\KPScript.exe"`).
+
+It takes a simple `String` as parameter to get the command line.
+
+Example:
+```ruby
+use_kpscript_from '/path/to/kpscript'
+```
+
+### `secrets_from_keepass`
+
+Define a KeePass database to read secrets from.
+A base group path of the KeePass database can also be specified to only read secrets from this group path.
+
+All entries, attachments and sub-groups from the base group will be read as secrets.
+
+Can be applied to subset of nodes using the [`for_nodes` DSL method](/docs/config_dsl.md#for_nodes).
+
+It takes the following parameters:
+* **database** (`String`): Database file path.
+* **group_path** (`Array<String>`): Group path to extract from [default: `[]`].
+
+Example:
+```ruby
+secrets_from_keepass(
+  database: '/path/to/database.kdbx',
+  group_path: %w[Secrets Automation]
+)
+```
+
+## Used credentials
+
+| Credential | Usage
+| --- | --- |
+| `keepass` | Used to get the password to the database. No need to be set if the database opens without password. |
+
+## Used Metadata
+
+| Metadata | Type | Usage
+| --- | --- | --- |
+
+## Used environment variables
+
+| Variable | Usage
+| --- | --- |
+| `hpc_key_file_for_keepass` | Optional path to the key file needed to open the database |
+| `hpc_password_enc_for_keepass` | Optional encrypted password needed to open the database |
+
+## External tools dependencies
+
+* [KeePass](https://keepass.info/) to open databases.
+* [KPScript KeePass plugin](https://keepass.info/plugins.html#kpscript) to query KeePass API.

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -9,6 +9,7 @@ require 'hybrid_platforms_conductor/logger_helpers'
 require 'hybrid_platforms_conductor/nodes_handler'
 require 'hybrid_platforms_conductor/services_handler'
 require 'hybrid_platforms_conductor/plugins'
+require 'hybrid_platforms_conductor/safe_merge'
 
 module HybridPlatformsConductor
 
@@ -500,34 +501,7 @@ module HybridPlatformsConductor
 
     private
 
-    # Safe-merge 2 hashes.
-    # Safe-merging is done by:
-    # * Merging values that are hashes.
-    # * Reporting errors when values conflict.
-    # When values are conflicting, the initial hash won't modify those conflicting values and will stop the merge.
-    #
-    # Parameters::
-    # * *hash* (Hash): Hash to be modified merging hash_to_merge
-    # * *hash_to_merge* (Hash): Hash to be merged into hash
-    # Result::
-    # * nil or Array<Object>: nil in case of success, or the keys path leading to a conflicting value in case of error
-    def safe_merge(hash, hash_to_merge)
-      conflicting_path = nil
-      hash_to_merge.each do |key, value_to_merge|
-        if hash.key?(key)
-          if hash[key].is_a?(Hash) && value_to_merge.is_a?(Hash)
-            sub_conflicting_path = safe_merge(hash[key], value_to_merge)
-            conflicting_path = [key] + sub_conflicting_path unless sub_conflicting_path.nil?
-          elsif hash[key] != value_to_merge
-            conflicting_path = [key]
-          end
-        else
-          hash[key] = value_to_merge
-        end
-        break unless conflicting_path.nil?
-      end
-      conflicting_path
-    end
+    include SafeMerge
 
     # Get the list of retriable errors a node got from deployment logs.
     # Useful to know if an error is non-deterministic (due to external and temporary factors).

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb

@@ -0,0 +1,173 @@
+require 'base64'
+require 'nokogiri'
+require 'tempfile'
+require 'keepass_kpscript'
+require 'zlib'
+require 'hybrid_platforms_conductor/credentials'
+require 'hybrid_platforms_conductor/safe_merge'
+require 'hybrid_platforms_conductor/secrets_reader'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Get secrets from a KeePass database
+      class Keepass < HybridPlatformsConductor::SecretsReader
+
+        include SafeMerge
+
+        # Extend the Config DSL
+        module ConfigDSLExtension
+
+          # List of defined KeePass secrets. Each info has the following properties:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+          # * *database* (String): Database file path.
+          # * *group_path* (Array<String>): Group path to extract from.
+          # Array< Hash<Symbol, Object> >
+          attr_reader :keepass_secrets
+
+          # String: The KPScript command line
+          attr_reader :kpscript
+
+          # Mixin initializer
+          def init_keepass_config
+            @keepass_secrets = []
+            @kpscript = nil
+          end
+
+          # Set the KPScript command line
+          #
+          # Parameters::
+          # * *cmd* (String): KPScript command line
+          def use_kpscript_from(cmd)
+            @kpscript = cmd
+          end
+
+          # Set a KeePass database configuration
+          #
+          # Parameters::
+          # * *database* (String): Database file path.
+          # * *group_path* (Array<String>): Group path to extract from [default: []].
+          def secrets_from_keepass(database:, group_path: [])
+            @keepass_secrets << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              database: database,
+              group_path: group_path
+            }
+          end
+
+        end
+
+        Config.extend_config_dsl_with ConfigDSLExtension, :init_keepass_config
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          secrets = {}
+          # As we are dealing with global secrets, cache the reading for performance between nodes and services.
+          # Keep secrets cache grouped by URL/ID
+          @secrets = {} unless defined?(@secrets)
+          @nodes_handler.select_confs_for_node(node, @config.keepass_secrets).each do |keepass_secrets_info|
+            secret_id = "#{keepass_secrets_info[:database]}:#{keepass_secrets_info[:group_path].join('/')}"
+            unless @secrets.key?(secret_id)
+              raise 'Missing KPScript configuration. Please use use_kpscript_from to set it.' if @config.kpscript.nil?
+
+              Credentials.with_credentials_for(:keepass, @logger, @logger_stderr) do |_user, password|
+                Tempfile.create('hpc_keepass') do |xml_file|
+                  key_file = ENV['hpc_key_file_for_keepass']
+                  password_enc = ENV['hpc_password_enc_for_keepass']
+                  keepass_credentials = {}
+                  keepass_credentials[:password] = password if password
+                  keepass_credentials[:password_enc] = password_enc if password_enc
+                  keepass_credentials[:key_file] = key_file if key_file
+                  KeepassKpscript.
+                    use(@config.kpscript, debug: log_debug?).
+                    open(keepass_secrets_info[:database], **keepass_credentials).
+                    export('KeePass XML (2.x)', xml_file.path, group_path: keepass_secrets_info[:group_path].empty? ? nil : keepass_secrets_info[:group_path])
+                  @secrets[secret_id] = parse_xml_secrets(Nokogiri::XML(xml_file).at_xpath('KeePassFile/Root/Group'))
+                end
+              end
+            end
+            conflicting_path = safe_merge(secrets, @secrets[secret_id])
+            raise "Secret set at path #{conflicting_path.join('->')} by #{keepass_secrets_info[:database]}#{keepass_secrets_info[:group_path].empty? ? '' : " from group #{keepass_secrets_info[:group_path].join('/')}"} for service #{service} on node #{node} has conflicting values (#{log_debug? ? "#{@secrets[secret_id].dig(*conflicting_path)} != #{secrets.dig(*conflicting_path)}" : 'set debug for value details'})." unless conflicting_path.nil?
+          end
+          secrets
+        end
+
+        private
+
+        # List of fields to include in the secrets and their corresponding XML name
+        FIELDS = {
+          notes: 'Notes',
+          password: 'Password',
+          url: 'URL',
+          user_name: 'UserName'
+        }
+
+        # Parse XML secrets from a Nokogiri XML group node
+        #
+        # Parameters::
+        # * *group* (Nokogiri::XML::Element): The group to parse
+        # Result::
+        # * Hash: The JSON secrets parsed from this group
+        def parse_xml_secrets(group)
+          # Parse all entries
+          group.xpath('Entry').map do |entry|
+            [
+              entry.at_xpath('String/Key[contains(.,"Title")]/../Value').text,
+              FIELDS.map do |property, field|
+                value = entry.at_xpath("String/Key[contains(.,\"#{field}\")]/../Value")&.text
+                if value.nil? || value.empty?
+                  nil
+                else
+                  [
+                    property.to_s,
+                    value
+                  ]
+                end
+              end.compact.to_h.merge(
+                entry.xpath('Binary').map do |binary|
+                  binary_meta = group.document.at_xpath("KeePassFile/Meta/Binaries/Binary[@ID=#{Integer(binary.xpath('Value').attr('Ref').value)}]")
+                  binary_content = Base64.decode64(binary_meta.text)
+                  if binary_meta.attr('Compressed') == 'True'
+                    gz = Zlib::GzipReader.new(StringIO.new(binary_content))
+                    binary_content = gz.read
+                    gz.close
+                  end
+                  [
+                    binary.xpath('Key').text,
+                    binary_content
+                  ]
+                end.to_h
+              )
+            ]
+          end.to_h.merge(
+            # Add children groups
+            group.xpath('Group').map do |sub_group|
+              [
+                sub_group.at_xpath('Name').text,
+                parse_xml_secrets(sub_group)
+              ]
+            end.to_h
+          )
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/plugins.rb

@@ -1,3 +1,4 @@
+require 'forwardable'
 require 'hybrid_platforms_conductor/logger_helpers'
 
 module HybridPlatformsConductor

data/lib/hybrid_platforms_conductor/safe_merge.rb

@@ -0,0 +1,37 @@
+module HybridPlatformsConductor
+
+  # Provide an easy way to safe-merge hashes
+  module SafeMerge
+
+    # Safe-merge 2 hashes.
+    # Safe-merging is done by:
+    # * Merging values that are hashes.
+    # * Reporting errors when values conflict.
+    # When values are conflicting, the initial hash won't modify those conflicting values and will stop the merge.
+    #
+    # Parameters::
+    # * *hash* (Hash): Hash to be modified merging hash_to_merge
+    # * *hash_to_merge* (Hash): Hash to be merged into hash
+    # Result::
+    # * nil or Array<Object>: nil in case of success, or the keys path leading to a conflicting value in case of error
+    def safe_merge(hash, hash_to_merge)
+      conflicting_path = nil
+      hash_to_merge.each do |key, value_to_merge|
+        if hash.key?(key)
+          if hash[key].is_a?(Hash) && value_to_merge.is_a?(Hash)
+            sub_conflicting_path = safe_merge(hash[key], value_to_merge)
+            conflicting_path = [key] + sub_conflicting_path unless sub_conflicting_path.nil?
+          elsif hash[key] != value_to_merge
+            conflicting_path = [key]
+          end
+        else
+          hash[key] = value_to_merge
+        end
+        break unless conflicting_path.nil?
+      end
+      conflicting_path
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb

@@ -17,9 +17,11 @@ module HybridPlatformsConductor
           @topographer.force_cluster_strict_hierarchy
           # Write a Graphviz file
           File.open(file_name, 'w') do |f|
-            f.puts 'digraph unix {
-              size="6,6";
-              node [style=filled];'
+            f.puts <<~EO_GRAPHVIZ
+              digraph unix {
+                size="6,6";
+                node [style=filled];
+            EO_GRAPHVIZ
             # First write the definition of all nodes
             # Find all nodes belonging to no cluster
             orphan_nodes = @topographer.nodes_graph.keys

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '33.2.4'
+  VERSION = '33.3.0'
 
 end

data/spec/hybrid_platforms_conductor_test.rb

@@ -101,6 +101,10 @@ module HybridPlatformsConductorTest
         ENV.delete 'hpc_user_for_thycotic'
         ENV.delete 'hpc_password_for_thycotic'
         ENV.delete 'hpc_domain_for_thycotic'
+        ENV.delete 'hpc_user_for_keepass'
+        ENV.delete 'hpc_password_for_keepass'
+        ENV.delete 'hpc_password_enc_for_keepass'
+        ENV.delete 'hpc_key_file_for_keepass'
         ENV.delete 'hpc_certificates'
         ENV.delete 'hpc_interactive'
         ENV.delete 'hpc_test_cookbooks_path'

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb

@@ -13,10 +13,10 @@ describe HybridPlatformsConductor::ActionsExecutor do
       end
 
       it 'returns 1 defined gateway with its content' do
-        ssh_gateway = '
+        ssh_gateway = <<~EO_CONFIG
           Host gateway
             Hostname mygateway.com
-        '
+        EO_CONFIG
         with_repository do
           with_platforms "gateway :gateway_1, '#{ssh_gateway}'" do
             expect(test_config.ssh_for_gateway(:gateway_1)).to eq ssh_gateway
@@ -34,10 +34,12 @@ describe HybridPlatformsConductor::ActionsExecutor do
 
       it 'returns several defined gateways' do
         with_repository do
-          with_platforms '
-            gateway :gateway_1, \'\'
-            gateway :gateway_2, \'\'
-          ' do
+          with_platforms(
+            <<~EO_CONFIG
+              gateway :gateway_1, ''
+              gateway :gateway_2, ''
+            EO_CONFIG
+          ) do
             expect(test_config.known_gateways.sort).to eq %i[gateway_1 gateway_2].sort
           end
         end

data/spec/hybrid_platforms_conductor_test/api/config_spec.rb

@@ -19,10 +19,12 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'returns several defined OS images' do
-    with_platforms '
-      os_image :image_1, \'/path/to/image_1\'
-      os_image :image_2, \'/path/to/image_2\'
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        os_image :image_1, '/path/to/image_1'
+        os_image :image_2, '/path/to/image_2'
+      EO_CONFIG
+    ) do
       expect(test_config.known_os_images.sort).to eq %i[image_1 image_2].sort
     end
   end
@@ -49,11 +51,13 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'includes several configuration files' do
-    with_platforms '
-      os_image :image_1, \'/path/to/image_1\'
-      include_config_from "#{__dir__}/my_conf_1.rb"
-      include_config_from "#{__dir__}/my_conf_2.rb"
-    ' do |hybrid_platforms_dir|
+    with_platforms(
+      <<~'EO_CONFIG'
+        os_image :image_1, '/path/to/image_1'
+        include_config_from "#{__dir__}/my_conf_1.rb"
+        include_config_from "#{__dir__}/my_conf_2.rb"
+      EO_CONFIG
+    ) do |hybrid_platforms_dir|
       File.write("#{hybrid_platforms_dir}/my_conf_1.rb", <<~'EO_CONFIG')
         os_image :image_4, '/path/to/image_4'
         include_config_from "#{__dir__}/my_conf_3.rb"
@@ -65,9 +69,7 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'applies nodes specific configuration to all nodes by default' do
-    with_platforms '
-      expect_tests_to_fail :my_test, \'Failure reason\'
-    ' do
+    with_platforms 'expect_tests_to_fail :my_test, \'Failure reason\'' do
       expect(test_config.expected_failures).to eq [
         {
           nodes_selectors_stack: [],
@@ -79,12 +81,14 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'filters nodes specific configuration to nodes sets in a scope' do
-    with_platforms '
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-      end
-      expect_tests_to_fail :my_test_2, \'Failure reason 2\'
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_1, 'Failure reason 1'
+        end
+        expect_tests_to_fail :my_test_2, 'Failure reason 2'
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -102,14 +106,16 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'filters nodes specific configuration in a scoped stack' do
-    with_platforms '
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-        for_nodes(%w[node2 node3 node4]) do
-          expect_tests_to_fail :my_test_2, \'Failure reason 2\'
+    with_platforms(
+      <<~EO_CONFIG
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_1, 'Failure reason 1'
+          for_nodes(%w[node2 node3 node4]) do
+            expect_tests_to_fail :my_test_2, 'Failure reason 2'
+          end
         end
-      end
-    ' do
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -127,13 +133,15 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'returns the expected failures correctly' do
-    with_platforms '
-      expect_tests_to_fail :my_test_1, \'Failure reason 1\'
-      expect_tests_to_fail %i[my_test_2 my_test_3], \'Failure reason 23\'
-      for_nodes(%w[node1 node2 node3]) do
-        expect_tests_to_fail :my_test_4, \'Failure reason 4\'
-      end
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        expect_tests_to_fail :my_test_1, 'Failure reason 1'
+        expect_tests_to_fail %i[my_test_2 my_test_3], 'Failure reason 23'
+        for_nodes(%w[node1 node2 node3]) do
+          expect_tests_to_fail :my_test_4, 'Failure reason 4'
+        end
+      EO_CONFIG
+    ) do
       sort_proc = proc { |expected_failure_info| expected_failure_info[:reason] }
       expect(test_config.expected_failures.sort_by(&sort_proc)).to eq [
         {
@@ -156,12 +164,14 @@ describe HybridPlatformsConductor::Config do
   end
 
   it 'returns the deployment schedules correctly' do
-    with_platforms '
-      deployment_schedule(IceCube::Schedule.new(Time.parse(\'2020-05-01 11:22:33 UTC\')))
-      for_nodes(%w[node1 node2 node3]) do
-        deployment_schedule(IceCube::Schedule.new(Time.parse(\'2020-05-02 22:33:44 UTC\')))
-      end
-    ' do
+    with_platforms(
+      <<~EO_CONFIG
+        deployment_schedule(IceCube::Schedule.new(Time.parse('2020-05-01 11:22:33 UTC')))
+        for_nodes(%w[node1 node2 node3]) do
+          deployment_schedule(IceCube::Schedule.new(Time.parse('2020-05-02 22:33:44 UTC')))
+        end
+      EO_CONFIG
+    ) do
       sort_proc = proc { |deployment_schedule_info| deployment_schedule_info[:schedule].to_ical }
       expect(test_config.deployment_schedules.sort_by(&sort_proc)).to eq [
         {