hybrid_platforms_conductor 32.13.4 → 32.14.0

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb

@@ -0,0 +1,51 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      class ServerlessChef < HybridPlatformsConductor::PlatformHandler
+
+        # Small class that can get a Ruby DSL file and return all DSL calls that have been made to it
+        class DslParser
+
+          # List of calls made by parsing the source file
+          # Array
+          attr_reader :calls
+
+          # Constructor
+          #
+          # Parameters::
+          # * *calls* (Array): List of calls to complement [default = []]
+          def initialize(calls = [])
+            @calls = calls
+          end
+
+          # Parse a file and get all its DSL calls
+          #
+          # Parameters::
+          # * *source* (String): File to parse
+          def parse(source)
+            instance_eval(File.read(source))
+          end
+
+          def method_missing(method_name, *args, &block)
+            sub_calls = []
+            @calls << {
+              method: method_name,
+              args: args,
+              block: block,
+              calls_on_result: sub_calls
+            }
+            DslParser.new(sub_calls)
+          end
+
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb

@@ -0,0 +1,271 @@
+require 'hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      class ServerlessChef < HybridPlatformsConductor::PlatformHandler
+
+        # Build the recipes tree from a ServerlessChef platform
+        class RecipesTreeBuilder
+
+          # Constructor
+          #
+          # Parameters::
+          # * *config* (Config): Configuration that can be used to tune tree building
+          # * *platform* (ServerlessChef): Platform for which we build the recipes tree
+          def initialize(config, platform)
+            @config = config
+            @platform = platform
+          end
+
+          # Get the whole tree of recipes
+          #
+          # Result::
+          # * The tree of recipes:
+          #   Hash< Symbol,   Hash< Symbol, Hash<Symbol,Object> >
+          #   Hash< cookbook, Hash< recipe, recipe_info         >
+          #   Each recipe info has the following attributes:
+          #   * *included_recipes* (Array< [String or nil, Symbol, Symbol] >): List of [cookbook_dir, cookbook, recipe] included by this recipe
+          #   * *used_by_policies* (Array<String>): List of policies that include (recursively) this recipe
+          #   * *used_templates* (Array<String>): List of template sources used by this recipe
+          #   * *used_files* (Array<String>): List of cookbook files used by this recipe
+          #   * *used_cookbooks* (Array<Symbol>): List of additional used cookbooks (for example for resources)
+          def full_recipes_tree
+            @recipes_tree = {}
+            @platform.deployable_services.each do |service|
+              policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
+                add_recipe_in_tree(cookbook_dir, cookbook, recipe)
+              end
+            end
+            @platform.deployable_services.each do |service|
+              policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
+                mark_recipe_used_by_policy(cookbook, recipe, service)
+              end
+            end
+            @recipes_tree
+          end
+
+          private
+
+          # Get the run list of a given policy
+          #
+          # Parameters::
+          # * *policy* (String): Policy to get the run list from
+          # Result::
+          # * Array<[String or nil, Symbol,Symbol]>: Run list of the given policy, as [cookbook_dir, cookbook, recipe]
+          def policy_run_list(policy)
+            # Read the policy file
+            dsl_parser = DslParser.new
+            policy_file = "#{@platform.repository_path}/policyfiles/#{policy}.rb"
+            dsl_parser.parse(policy_file)
+            run_list_call = dsl_parser.calls.find { |call_info| call_info[:method] == :run_list }
+            raise "Policy #{policy} has no run list defined in #{policy_file}" if run_list_call.nil?
+            run_list_call[:args].map { |recipe_def| decode_recipe(recipe_def) }
+          end
+
+          # Return the cookbook directory, cookbook name and recipe name from which a recipe definition is found.
+          # The following forms are handled:
+          # * cookbook
+          # * cookbook::recipe
+          # * recipe[cookbook]
+          # * recipe[cookbook::recipe]
+          #
+          # Parameters::
+          # * *recipe_def* (String): Recipe definition (cookbook or cookbook::recipe).
+          # Result::
+          # * String: The cookbook directory, or nil if unknown
+          # * Symbol: The cookbook name
+          # * Symbol: The recipe name
+          def decode_recipe(recipe_def)
+            recipe_def = $1 if recipe_def =~ /^recipe\[(.+)\]$/
+            cookbook, recipe = recipe_def.split('::').map(&:to_sym)
+            recipe = :default if recipe.nil?
+            # Find the cookbook it belongs to
+            cookbook_dir = @platform.known_cookbook_paths.find { |cookbook_path| File.exist?("#{@platform.repository_path}/#{cookbook_path}/#{cookbook}") }
+            raise "Unknown recipe #{cookbook}::#{recipe} from cookbook #{@platform.repository_path}/#{cookbook_dir}/#{cookbook}." if !cookbook_dir.nil? && !File.exist?("#{@platform.repository_path}/#{cookbook_dir}/#{cookbook}/recipes/#{recipe}.rb")
+            return cookbook_dir, cookbook, recipe
+          end
+
+          # Fill the tree with a recipe and all its dependencies
+          #
+          # Parameters::
+          # * *cookbook_dir* (String): The cookbook directory, or nil if unknown
+          # * *cookbook* (Symbol): The cookbook name
+          # * *recipe* (Symbol): The recipe name
+          def add_recipe_in_tree(cookbook_dir, cookbook, recipe)
+            @recipes_tree[cookbook] = {} unless @recipes_tree.key?(cookbook)
+            unless @recipes_tree[cookbook].key?(recipe)
+              recipe_info =
+                if cookbook_dir.nil?
+                  # This recipe comes from an external cookbook, we won't get into it.
+                  {
+                    included_recipes: [],
+                    used_templates: [],
+                    used_files: [],
+                    used_cookbooks: []
+                  }
+                else
+                  recipe_usage(cookbook_dir, cookbook, recipe)
+                end
+              @recipes_tree[cookbook][recipe] = recipe_info.merge(
+                used_by_policies: []
+              )
+              recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
+                add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
+              end
+            end
+          end
+
+          # Get some info on a given recipe.
+          # Parses for:
+          # * include_recipe.
+          # * source of template and cookbook_file.
+          # * Any library helper we know use some recipes.
+          # * Any resource we have defined in other cookbooks.
+          # * Any library method we have defined in other cookbooks.
+          #
+          # Parameters::
+          # * *cookbook_dir* (String): The cookbook directory
+          # * *cookbook* (Symbol): The cookbook name
+          # * *recipe* (Symbol): The recipe name
+          # Result::
+          # * Hash<Symbol,Object>: A structure describing the recipe:
+          #   * *included_recipes* (Array< [String, Symbol, Symbol] >): List of tuples [cookbook_dir, cookbook, recipe] used by this recipe
+          #   * *used_templates* (Array<String>): List of template sources used by this recipe
+          #   * *used_files* (Array<String>): List of cookbook files used by this recipe
+          #   * *used_cookbooks* (Array<String>): List of additional cookbooks used by this recipe
+          def recipe_usage(cookbook_dir, cookbook, recipe)
+            recipe_content = File.read("#{@platform.repository_path}/#{cookbook_dir}/#{cookbook}/recipes/#{recipe}.rb")
+            # Check for include_recipe
+            used_recipes = recipe_content.
+              scan(/include_recipe\s+["'](\w+(::\w+)?)["']/).
+              map { |(recipe_def, _sub_grp)| decode_recipe(recipe_def) }
+            # Check for some helpers we know include some recipes
+            @config.known_helpers_including_recipes.each do |helper_name, used_recipes_by_helper|
+              if recipe_content =~ Regexp.new(/(\W|^)#{Regexp.escape(helper_name)}(\W|$)/)
+                used_recipes.concat(used_recipes_by_helper.map { |recipe_def| decode_recipe(recipe_def) })
+                used_recipes.uniq!
+              end
+            end
+            sources = []
+            recipe_content.
+              scan(/source\s+(["'])(.+?)\1/).
+              each do |(_sub_grp, source)|
+                sources << source unless source =~ /^https?:\/\//
+              end
+            erb_sources = sources.select { |source| File.extname(source).downcase == '.erb' }
+            non_erb_sources = sources - erb_sources
+            erb_sources.concat(recipe_content.scan(/template:?\s+(["'])(.+?)\1/).map { |(_sub_grp, source)| source })
+            # Check for known resources and library methods
+            used_cookbooks = []
+            known_resources.each do |cookbook, methods|
+              used_cookbooks << cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
+            end
+            known_library_methods.each do |cookbook, methods|
+              used_cookbooks << cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
+            end
+            {
+              included_recipes: used_recipes,
+              used_templates: erb_sources,
+              used_files: non_erb_sources,
+              used_cookbooks: used_cookbooks.uniq
+            }
+          end
+
+          # Get the user defined resources, per cookbook.
+          # Keep a memory cache of it.
+          #
+          # Result::
+          # * Hash< Symbol, Array<String> >: List of resource names (as useable methods), per cookbook
+          def known_resources
+            unless defined?(@known_resources)
+              @known_resources = {}
+              for_each_cookbook do |cookbook, cookbook_dir|
+                if File.exist?("#{cookbook_dir}/resources")
+                  @known_resources[cookbook] = Dir.glob("#{cookbook_dir}/resources/*.rb").map do |resource_file|
+                    "#{cookbook}_#{File.basename(resource_file, '.rb')}"
+                  end
+                end
+              end
+            end
+            @known_resources
+          end
+
+          # Get the list of library methods we know we have to ignore from the parsing
+          # Array<String>
+          INVALID_LIBRARY_METHODS = [
+            'initialize'
+          ]
+
+          # Get the user defined library methods, per cookbook.
+          # Keep a memory cache of it.
+          #
+          # Result::
+          # * Hash< Symbol, Array<String> >: List of library method names, per cookbook
+          def known_library_methods
+            unless defined?(@known_library_methods)
+              @known_library_methods = {}
+              for_each_cookbook do |cookbook, cookbook_dir|
+                if File.exist?("#{cookbook_dir}/libraries")
+                  found_methods =  Dir.glob("#{cookbook_dir}/libraries/*.rb").
+                      map { |lib_file| File.read(lib_file).scan(/\bdef\s+(\w+)\b/).map { |(method_name)| method_name } }.
+                      flatten - INVALID_LIBRARY_METHODS
+                  @known_library_methods[cookbook] = found_methods unless found_methods.empty?
+                end
+              end
+            end
+            @known_library_methods
+          end
+
+          # Iterate over all cookbooks
+          #
+          # Parameters::
+          # * Proc: Code called for each cookbook:
+          #   * Parameters::
+          #     * *cookbook* (Symbol): Cookbook name
+          #     * *cookbook_dir* (String): Cookbook directory
+          def for_each_cookbook
+            @platform.known_cookbook_paths.each do |cookbook_path|
+              cookbooks_in(cookbook_path).each do |cookbook, cookbook_dir|
+                yield cookbook, cookbook_dir
+              end
+            end
+          end
+
+          # Get the list of cookbooks of a given cookbook type
+          #
+          # Parameters::
+          # * *cookbook_type* (String): The cookbook type (like site-cookbook)
+          # Result::
+          # * Hash<Symbol, String>: List of cookbook directories, per cookbook name
+          def cookbooks_in(cookbook_type)
+            Hash[Dir.glob("#{@platform.repository_path}/#{cookbook_type}/*").map { |dir| [File.basename(dir).to_sym, dir] }.sort]
+          end
+
+          # Mark a recipe (and its included recipes) as used by a policy
+          #
+          # Parameters::
+          # * *cookbook* (Symbol): The cookbook
+          # * *recipe* (Symbol): The recipe
+          # * *used_by_policy* (String): The policy using this recipe
+          def mark_recipe_used_by_policy(cookbook, recipe, used_by_policy)
+            unless @recipes_tree[cookbook][recipe][:used_by_policies].include?(used_by_policy)
+              @recipes_tree[cookbook][recipe][:used_by_policies] << used_by_policy
+              @recipes_tree[cookbook][recipe][:included_recipes].each do |(_sub_cookbook_dir, sub_cookbook, sub_recipe)|
+                mark_recipe_used_by_policy(sub_cookbook, sub_recipe, used_by_policy)
+              end
+            end
+          end
+
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/nodes_handler.rb

@@ -269,12 +269,16 @@ module HybridPlatformsConductor
     #
     # Parameters::
     # * *node* (String): Node
-    # * *property* (Symbol): The property name
+    # * *property* (Symbol or nil): The property name, or nil for all [default=nil]
     # Result::
-    # * Object or nil: The node's metadata value for this property, or nil if none
-    def metadata_of(node, property)
-      prefetch_metadata_of([node], property) unless @metadata.key?(node) && @metadata[node].key?(property)
-      @metadata[node][property]
+    # * Object or nil: The node's metadata value for this property, or nil if none, or a Hash of metadata if property was nil
+    def metadata_of(node, property = nil)
+      if property.nil?
+        @metadata[node] || {}
+      else
+        prefetch_metadata_of([node], property) unless @metadata.key?(node) && @metadata[node].key?(property)
+        @metadata[node][property]
+      end
     end
 
     # Override a metadata property for a given node

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.13.4'
+  VERSION = '32.14.0'
 
 end

data/spec/hybrid_platforms_conductor_test.rb

@@ -35,6 +35,7 @@ require 'hybrid_platforms_conductor_test/helpers/platforms_handler_helpers'
 require 'hybrid_platforms_conductor_test/helpers/plugins_helpers'
 require 'hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers'
 require 'hybrid_platforms_conductor_test/helpers/reports_handler_helpers'
+require 'hybrid_platforms_conductor_test/helpers/serverless_chef_helpers'
 require 'hybrid_platforms_conductor_test/helpers/services_handler_helpers'
 require 'hybrid_platforms_conductor_test/helpers/tests_runner_helpers'
 require 'hybrid_platforms_conductor_test/platform_handler_plugins/test'
@@ -69,6 +70,7 @@ module HybridPlatformsConductorTest
     include PluginsHelpers
     include ProvisionerProxmoxHelpers
     include ReportsHandlerHelpers
+    include ServerlessChefHelpers
     include ServicesHandlerHelpers
     include TestsRunnerHelpers
 
@@ -95,6 +97,7 @@ module HybridPlatformsConductorTest
         ENV.delete 'hpc_domain_for_thycotic'
         ENV.delete 'hpc_certificates'
         ENV.delete 'hpc_interactive'
+        ENV.delete 'hpc_test_cookbooks_path'
         # Set the necessary Hybrid Platforms Conductor environment variables
         ENV['hpc_ssh_user'] = 'test_user'
         HybridPlatformsConductor::ServicesHandler.packaged_deployments.clear

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb

@@ -84,6 +84,29 @@ describe HybridPlatformsConductor::Deployer do
       end
     end
 
+    it 'gives a new test instance ready to be used in place of the node without local node' do
+      with_test_platform(
+        {
+          nodes: {
+            'node1' => { meta: { local_node: true } },
+            'node2' => { meta: { local_node: true } }
+          }
+        }
+      ) do |repository|
+        register_plugins(:provisioner, { test_provisioner: HybridPlatformsConductorTest::TestProvisioner })
+        File.write("#{test_config.hybrid_platforms_dir}/dummy_secrets.json", '{}')
+        HybridPlatformsConductorTest::TestProvisioner.mocked_states = %i[created created running exited]
+        HybridPlatformsConductorTest::TestProvisioner.mocked_ip = '172.17.0.1'
+        expect(Socket).to receive(:tcp).with('172.17.0.1', 22, { connect_timeout: 1 }) do |&block|
+          block.call
+        end
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node1', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.instance_eval { @nodes_handler.get_local_node_of('node1') }).to eq false
+          expect(sub_test_deployer.instance_eval { @nodes_handler.get_local_node_of('node2') }).to eq true
+        end
+      end
+    end
+
     it 'gives a new test instance ready to be used in place of the node without sudo specificities' do
       with_test_platform(
         {

data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb

@@ -38,6 +38,17 @@ describe HybridPlatformsConductor::NodesHandler do
       end
     end
 
+    it 'returns nodes currently known metadata using generic method' do
+      with_cmdb_test_platform do
+        test_nodes_handler.metadata_of('node1', :upcase)
+        test_nodes_handler.get_double_of('node1')
+        expect(test_nodes_handler.metadata_of('node1')).to eq(
+          double: 'node1node1',
+          upcase: 'NODE1'
+        )
+      end
+    end
+
     it 'returns nodes metadata using dynamic method several times (as the method is created dynamically)' do
       with_cmdb_test_platform do
         3.times { expect(test_nodes_handler.get_upcase_of('node1')).to eq 'NODE1' }

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb

@@ -0,0 +1,17 @@
+describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
+
+  context 'checking config DSL' do
+
+    it 'defines helpers that include recipes' do
+      with_repository do |repository|
+        with_platforms('helpers_including_recipes(my_helper: [\'cookbook1::recipe1\', \'cookbook2\'])') do
+          expect(test_config.known_helpers_including_recipes).to eq(
+            my_helper: %w[cookbook1::recipe1 cookbook2]
+          )
+        end
+      end
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb

@@ -0,0 +1,94 @@
+describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
+
+  context 'checking how deployment output is parsed' do
+
+    it 'parses a deployment output properly' do
+      with_serverless_chef_platforms('empty') do |platform|
+        stdout = <<~EOS
+          Starting Chef Client, version 14.14.29
+          resolving cookbooks for run list: ["policy_xae_websql::xae"]
+          Synchronizing Cookbooks:
+            - policy_xae_websql (0.1.0)
+            - chef-ruby (0.1.2)
+            - nginx (10.1.0)
+          Installing Cookbook Gems:
+          Compiling Cookbooks...
+          Recipe: site_debian::default
+            * apt_update[apt update] action update
+              - force update new lists of packages
+              * directory[/etc/apt/apt.conf.d] action create (up to date)
+              * file[/etc/apt/apt.conf.d/15update-stamp] action create_if_missing (up to date)
+              * execute[apt-get -q update] action run
+                - execute ["apt-get", "-q", "update"]
+            
+            Converging 145 resources
+          Recipe: policy_xae_websql::api
+            * site_artifactory_dpkg_package[xaecalcite] action install
+              * remote_file[/opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb] action create
+                - create new file /opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb
+                - update content in file /opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb from none to 39b0ca
+                (file sizes exceed 10000000 bytes, diff output suppressed)
+              * dpkg_package[xaecalcite] action install
+                - install version 0.2.4-1 of package xaecalcite
+            
+            * service[/var/lib/xaecalcite/xaecalcite.service] action enable (skipped due to not_if)
+
+          Running handlers:
+          Running handlers complete
+          Chef Client finished, 16/300 resources updated in 27 seconds
+        EOS
+        expect(platform.parse_deploy_output(stdout, '')). to eq [
+          {
+            action: 'update',
+            diffs: "force update new lists of packages\n",
+            name: 'apt_update[apt update]',
+            status: :changed
+          },
+          {
+            action: 'create (up to date)',
+            name: 'directory[/etc/apt/apt.conf.d]',
+            status: :identical
+          },
+          {
+            action: 'create_if_missing (up to date)',
+            name: 'file[/etc/apt/apt.conf.d/15update-stamp]',
+            status: :identical
+          },
+          {
+            action: 'run',
+            diffs: "execute [\"apt-get\", \"-q\", \"update\"]\n",
+            name: 'execute[apt-get -q update]',
+            status: :changed
+          },
+          {
+            action: 'install',
+            name: 'site_artifactory_dpkg_package[xaecalcite]',
+            status: :identical
+          },
+          {
+            action: 'create',
+            diffs: <<~EOS,
+              create new file /opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb
+              update content in file /opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb from none to 39b0ca
+            EOS
+            name: 'remote_file[/opt/chef_cache/xaecalcite_0.2.4-1_amd64.deb]',
+            status: :changed
+          },
+          {
+            action: 'install',
+            diffs: "install version 0.2.4-1 of package xaecalcite\n",
+            name: 'dpkg_package[xaecalcite]',
+            status: :changed
+          },
+          {
+            action: 'enable (skipped due to not_if)',
+            name: 'service[/var/lib/xaecalcite/xaecalcite.service]',
+            status: :identical
+          }
+        ]
+      end
+    end
+
+  end
+
+end