hybrid_platforms_conductor 32.13.4 → 32.16.2

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb

@@ -0,0 +1,317 @@
+describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
+
+  context 'checking files diff impacts' do
+
+    it 'returns no impact for no diffs' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from({})).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'ignores files with no impact' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/README.md' => {}
+        )).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns all nodes impact for global files' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'global.rb' => {}
+        )).to eq [
+          [],
+          [],
+          true
+        ]
+      end
+    end
+
+    it 'returns direct impacted nodes' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'nodes/node1.json' => {}
+        )).to eq [
+          %w[node1],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns direct impacted nodes with strange characters' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'nodes/node-v45.env_@user.json' => {}
+        )).to eq [
+          ['node-v45.env_@user'],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a change in its recipes' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a change in its attributes' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/attributes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a change in an included template' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          template '/home/file' do
+            source 'test_template.erb'
+          end
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/templates/default/test_template.erb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'does not return impacted service due to a change in a non included template' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/templates/default/test_template.erb' => {}
+        )).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a change in an included file' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          file '/home/file' do
+            source 'test_file'
+          end
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/files/default/test_file' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'does not return impacted service due to a change in a non included file' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_1/files/default/test_file' => {}
+        )).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a resource usage in a recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          test_cookbook_2_my_resource
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/resources/my_resource.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'does not return impacted service due to a resource not being used in a recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/resources/my_resource.rb' => {}
+        )).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a library helper usage in a recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          a = my_library_helper(42)
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/libraries/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'ignored impacted service from an unknown helper' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          a = unknown_helper(42)
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_2],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to an unknown library helper usage that has been configured' do
+      with_serverless_chef_platforms(
+        'recipes',
+        additional_config: <<~EOS
+          helpers_including_recipes(unknown_helper: ['test_cookbook_2'])
+        EOS
+      ) do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          a = unknown_helper(42)
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1 test_policy_2],
+          false
+        ]
+      end
+    end
+
+    it 'does not return impacted service due to a library helper not being used in a recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/libraries/default.rb' => {}
+        )).to eq [
+          [],
+          [],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a usage of another cookbook\'s default recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          include_recipe 'test_cookbook_2'
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1 test_policy_2],
+          false
+        ]
+      end
+    end
+
+    it 'returns impacted service due to a usage of another cookbook\'s recipe' do
+      with_serverless_chef_platforms('recipes') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          include_recipe 'test_cookbook_2::other_recipe'
+        EOS
+        expect(platform.impacts_from(
+          'cookbooks/test_cookbook_2/recipes/other_recipe.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1],
+          false
+        ]
+      end
+    end
+
+    it 'ignores cookbooks from cookbook paths that are not configured' do
+      with_serverless_chef_platforms('several_cookbooks') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          include_recipe 'test_cookbook_2'
+        EOS
+        expect(platform.impacts_from(
+          'other_cookbooks/test_cookbook_2/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[],
+          true
+        ]
+      end
+    end
+
+    it 'considers cookbooks from non-standard cookbook paths that are configured' do
+      with_serverless_chef_platforms('several_cookbooks') do |platform, repository|
+        File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+          include_recipe 'test_cookbook_2'
+        EOS
+        ENV['hpc_test_cookbooks_path'] = 'other_cookbooks'
+        expect(platform.impacts_from(
+          'other_cookbooks/test_cookbook_2/recipes/default.rb' => {}
+        )).to eq [
+          [],
+          %w[test_policy_1 test_policy_2],
+          false
+        ]
+      end
+    end
+
+    it 'ignores cookbooks from cookbook paths that are configured but lie outside the platform' do
+      with_repository('other_cookbooks') do |other_repo|
+        FileUtils.mkdir_p("#{other_repo}/cookbooks/test_cookbook_2/recipes")
+        File.write("#{other_repo}/cookbooks/test_cookbook_2/recipes/default.rb", '')
+        with_serverless_chef_platforms('several_cookbooks') do |platform, repository|
+          File.write("#{repository}/cookbooks/test_cookbook_1/recipes/default.rb", <<~EOS)
+            include_recipe 'test_cookbook_2'
+          EOS
+          ENV['hpc_test_cookbooks_path'] = "#{other_repo}:other_cookbooks"
+          expect(platform.impacts_from(
+            'unknown_cookbooks/test_cookbook_2/recipes/default.rb' => {}
+          )).to eq [
+            [],
+            %w[],
+            true
+          ]
+        end
+      end
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb

@@ -0,0 +1,65 @@
+describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
+
+  context 'checking inventory' do
+
+    context 'with an empty platform' do
+
+      it 'returns no node' do
+        with_serverless_chef_platforms('empty') do |platform|
+          expect(platform.known_nodes).to eq []
+        end
+      end
+
+      it 'returns no nodes list' do
+        with_serverless_chef_platforms('empty') do |platform|
+          expect(platform.respond_to?(:known_nodes_lists)).to eq false
+        end
+      end
+
+      it 'returns no deployable services' do
+        with_serverless_chef_platforms('empty') do |platform|
+          expect(platform.deployable_services).to eq []
+        end
+      end
+
+    end
+
+    context 'with a platform having 1 node' do
+
+      it 'returns the node' do
+        with_serverless_chef_platforms('1_node') do |platform|
+          expect(platform.known_nodes).to eq ['node']
+        end
+      end
+
+      it 'returns correct metadata for this node' do
+        with_serverless_chef_platforms('1_node') do |platform|
+          expect(platform.metadata_for('node')).to eq(
+            description: 'Single test node',
+            image: 'debian_9',
+            private_ips: ['172.16.0.1'],
+            property1: {
+              'property11' => 'value11'
+            },
+            property2: 'value2'
+          )
+        end
+      end
+
+      it 'returns correct service for this node' do
+        with_serverless_chef_platforms('1_node') do |platform|
+          expect(platform.services_for('node')).to eq %w[test_policy]
+        end
+      end
+
+      it 'returns deployable services' do
+        with_serverless_chef_platforms('1_node') do |platform|
+          expect(platform.deployable_services).to eq %w[test_policy]
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb

@@ -0,0 +1,293 @@
+describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
+
+  context 'checking services packaging' do
+
+    # Expect a repository to be packaged and mock it
+    #
+    # Parameters::
+    # * *repository* (String): Repository to be packaged
+    # * *policy* (String): Expected policy to be packaged [default: 'test_policy']
+    # * *policy_file* (String): Expected policy file used [default: "policyfiles/#{policy}.rb"]
+    # * *install* (Boolean): Are we expecting the chef install stage? [default: true]
+    # * *export* (Boolean): Are we expecting the chef export stage? [default: true]
+    # * *data_bags* (Boolean): Do we expect data bags copy? [default: false]
+    # * *env* (String): Expected environment being packaged [default: 'prod']
+    # * Proc: Code called with mock in place
+    def with_packaging_mocked(
+      repository,
+      policy: 'test_policy',
+      policy_file: "policyfiles/#{policy}.rb",
+      install: true,
+      export: true,
+      data_bags: false,
+      env: 'prod'
+    )
+      with_cmd_runner_mocked(
+        if install
+          [
+            [
+              "cd #{repository} && /opt/chef-workstation/bin/chef install #{policy_file}",
+              proc do
+                # Mock the run_list stored in the lock file
+                File.write(
+                  "#{repository}/#{policy_file.gsub(/.rb$/, '.lock.json')}",
+                  {
+                    run_list: eval("[#{File.read("#{repository}/#{policy_file}").split("\n").select { |line| line =~ /^run_list.+$/ }.last.match(/^run_list(.+)$/)[1]}]").flatten
+                  }.to_json
+                )
+                [0, 'Chef install done', '']
+              end
+            ]
+          ]
+        else
+          []
+        end +
+        if export
+          [
+            ['whoami', proc { [0, 'test_user', ''] }, { optional: true }],
+            [
+              /^cd #{Regexp.escape(repository)} &&\s+sudo rm -rf dist\/#{Regexp.escape(env)}\/#{Regexp.escape(policy)} &&\s+\/opt\/chef-workstation\/bin\/chef export #{Regexp.escape(policy_file)} dist\/#{Regexp.escape(env)}\/#{Regexp.escape(policy)}#{data_bags ? " && cp -ar data_bags/ dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)}/" : ''}$/,
+              proc do
+                FileUtils.mkdir_p "#{repository}/dist/#{env}/#{policy}"
+                FileUtils.cp_r("#{repository}/data_bags", "#{repository}/dist/#{env}/#{policy}/") if data_bags
+                [0, 'Chef export done', '']
+              end
+            ]
+          ]
+        else
+          []
+        end
+      ) do
+        yield
+      end
+    end
+
+    context 'with an empty platform' do
+
+      it 'packages the repository doing nothing' do
+        with_serverless_chef_platforms('empty') do |platform, repository|
+          with_cmd_runner_mocked([]) do
+            platform.package(services: {}, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+    end
+
+    context 'with a platform having 1 node' do
+
+      it 'packages the repository for a given node and service' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository without resolving dependencies when the lock file already exists' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          File.write("#{repository}/policyfiles/test_policy.lock.json", '{}')
+          with_packaging_mocked(repository, install: false) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository with secrets' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: { secret: 'value' }, local_environment: false)
+            secret_file = "#{repository}/dist/prod/test_policy/data_bags/hpc_secrets/hpc_secrets.json"
+            expect(File.exist?(secret_file)).to eq true
+            expect(JSON.parse(File.read(secret_file))).to eq(
+              'id' => 'hpc_secrets',
+              'secret' => 'value'
+            )
+          end
+        end
+      end
+
+      it 'packages the repository for a given node and service in local mode' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => ['recipe[test_cookbook]'])
+          end
+        end
+      end
+
+      it 'packages the repository without resolving dependencies when the lock file already exists' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          File.write("#{repository}/policyfiles/test_policy.lock.json", '{}')
+          with_packaging_mocked(repository, install: false) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'does not package the repository twice for the same config' do
+        with_serverless_chef_platforms('1_node', as_git: true) do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+          with_cmd_runner_mocked([]) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository twice when the platform is not taken from git' do
+        with_serverless_chef_platforms('1_node') do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+          # Wait 2 seconds so that we are sure later Time.now will return different timestamps
+          sleep 2
+          with_packaging_mocked(repository, install: false) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository twice when the platform needs different secrets' do
+        with_serverless_chef_platforms('1_node', as_git: true) do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: { secret: 'value1' }, local_environment: false)
+          end
+          with_packaging_mocked(repository, install: false) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: { secret: 'value2' }, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository twice when the platform has new local files' do
+        with_serverless_chef_platforms('1_node', as_git: true) do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+          # Make sure we clean the cache (this mocks another Platform Handler instance running)
+          platform.remove_instance_variable :@info
+          with_packaging_mocked(repository, install: false) do
+            File.write("#{repository}/new_file", 'New file')
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages the repository twice when the platform has modified local files' do
+        with_serverless_chef_platforms('1_node', as_git: true) do |platform, repository|
+          with_packaging_mocked(repository) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+          # Wait 2 seconds so that we are sure the modified file will return a different timestamp
+          sleep 2
+          with_packaging_mocked(repository, install: false) do
+            File.write("#{repository}/chef_versions.yml", File.read("#{repository}/chef_versions.yml") + "\n\n")
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+    end
+
+    context 'with a platform having several nodes' do
+
+      it 'packages 1 service independently from another' do
+        with_serverless_chef_platforms('several_nodes') do |platform, repository|
+          with_packaging_mocked(repository, policy: 'test_policy_1') do
+            platform.package(services: { 'node1' => %w[test_policy_1] }, secrets: {}, local_environment: false)
+          end
+          with_packaging_mocked(repository, policy: 'test_policy_2') do
+            platform.package(services: { 'node2' => %w[test_policy_2] }, secrets: {}, local_environment: false)
+          end
+          with_cmd_runner_mocked([]) do
+            platform.package(services: { 'node1' => %w[test_policy_1] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+      it 'packages 1 service independently of the node on which it is to be deployed' do
+        with_serverless_chef_platforms('several_nodes') do |platform, repository|
+          with_packaging_mocked(repository, policy: 'test_policy_1') do
+            platform.package(services: { 'node1' => %w[test_policy_1] }, secrets: {}, local_environment: false)
+          end
+          with_cmd_runner_mocked([]) do
+            platform.package(services: { 'node2' => %w[test_policy_1] }, secrets: {}, local_environment: false)
+          end
+        end
+      end
+
+    end
+
+    context 'with a platform having data bags' do
+
+      it 'packages data bags' do
+        with_serverless_chef_platforms('data_bags') do |platform, repository|
+          with_packaging_mocked(repository, data_bags: true) do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
+            data_bag_file = "#{repository}/dist/prod/test_policy/data_bags/my_bag/my_item.json"
+            expect(File.exist?(data_bag_file)).to eq true
+            expect(JSON.parse(File.read(data_bag_file))).to eq(
+              'id' => 'my_item',
+              'content' => 'Bag content'
+            )
+          end
+        end
+      end
+
+    end
+
+    context 'with a platform having hpc_test cookbook' do
+
+      it 'packages the repository with before_run and after_run recipes wrapping the run list' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'hpc_test::before_run',
+              'recipe[test_cookbook]',
+              'hpc_test::after_run'
+            ])
+          end
+        end
+      end
+
+      it 'packages the repository with before_run only recipe' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          File.unlink "#{repository}/cookbooks/hpc_test/recipes/after_run.rb"
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'hpc_test::before_run',
+              'recipe[test_cookbook]'
+            ])
+          end
+        end
+      end
+
+      it 'packages the repository with after_run only recipe' do
+        with_serverless_chef_platforms('hpc_test') do |platform, repository|
+          File.unlink "#{repository}/cookbooks/hpc_test/recipes/before_run.rb"
+          with_packaging_mocked(repository, policy_file: 'policyfiles/test_policy.local.rb', env: 'local') do
+            platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: true)
+            local_policy_file = "#{repository}/policyfiles/test_policy.local.lock.json"
+            expect(File.exist?(local_policy_file)).to eq true
+            expect(JSON.parse(File.read(local_policy_file))).to eq('run_list' => [
+              'recipe[test_cookbook]',
+              'hpc_test::after_run'
+            ])
+          end
+        end
+      end
+
+    end
+
+  end
+
+end