chemlab 0.0.1

data/qa/specs/features/ee/browser_ui/secure/security_reports_spec.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module QA
+  context 'Secure', :docker, :runner, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/213676', type: :investigating } do
+    let(:number_of_dependencies_in_fixture) { 7 }
+    let(:dependency_scan_example_vuln) { 'Prototype pollution attack in mixin-deep' }
+    let(:container_scan_example_vuln) { 'CVE-2017-18269 in glibc' }
+    let(:sast_scan_example_vuln) { 'Cipher with no integrity' }
+    let(:dast_scan_example_vuln) { 'Cookie Without SameSite Attribute' }
+
+    describe 'Security Reports' do
+      after(:all) do
+        @runner.remove_via_api!
+      end
+
+      before(:all) do
+        @executor = "qa-runner-#{Time.now.to_i}"
+
+        Flow::Login.sign_in
+
+        @project = Resource::Project.fabricate_via_api! do |p|
+          p.name = Runtime::Env.auto_devops_project_name || 'project-with-secure'
+          p.description = 'Project with Secure'
+        end
+
+        @runner = Resource::Runner.fabricate! do |runner|
+          runner.project = @project
+          runner.name = @executor
+          runner.tags = %w[qa test]
+        end
+
+        # Push fixture to generate Secure reports
+        Resource::Repository::ProjectPush.fabricate! do |push|
+          push.project = @project
+          push.directory = Pathname
+            .new(__dir__)
+            .join('../../../../../ee/fixtures/secure_premade_reports')
+          push.commit_message = 'Create Secure compatible application to serve premade reports'
+        end.project.visit!
+
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success)
+      end
+
+      before do
+        Flow::Login.sign_in_unless_signed_in
+        @project.visit!
+      end
+
+      it 'displays security reports in the pipeline' do
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline)
+
+        Page::Project::Pipeline::Show.perform do |pipeline|
+          pipeline.click_on_security
+
+          filter_report_and_perform(pipeline, "Dependency Scanning") do
+            expect(pipeline).to have_vulnerability dependency_scan_example_vuln
+          end
+
+          filter_report_and_perform(pipeline, "Container Scanning") do
+            expect(pipeline).to have_vulnerability container_scan_example_vuln
+          end
+
+          filter_report_and_perform(pipeline, "SAST") do
+            expect(pipeline).to have_vulnerability sast_scan_example_vuln
+          end
+
+          filter_report_and_perform(pipeline, "DAST") do
+            expect(pipeline).to have_vulnerability dast_scan_example_vuln
+          end
+        end
+      end
+
+      it 'displays security reports in the project security dashboard' do
+        Page::Project::Menu.perform(&:click_project)
+        Page::Project::Menu.perform(&:click_on_security_dashboard)
+
+        EE::Page::Project::Secure::Show.perform do |dashboard|
+          filter_report_and_perform(dashboard, "Dependency Scanning") do
+            expect(dashboard).to have_vulnerability dependency_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "Container Scanning") do
+            expect(dashboard).to have_vulnerability container_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "SAST") do
+            expect(dashboard).to have_vulnerability sast_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "DAST") do
+            expect(dashboard).to have_vulnerability dast_scan_example_vuln
+          end
+        end
+      end
+
+      it 'displays security reports in the group security dashboard' do
+        Page::Main::Menu.perform(&:go_to_groups)
+        Page::Dashboard::Groups.perform do |groups|
+          groups.click_group @project.group.path
+        end
+        Page::Group::Menu.perform(&:click_group_security_link)
+
+        EE::Page::Group::Secure::Show.perform do |dashboard|
+          dashboard.filter_project(@project.name)
+
+          filter_report_and_perform(dashboard, "Dependency Scanning") do
+            expect(dashboard).to have_vulnerability dependency_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "Container Scanning") do
+            expect(dashboard).to have_vulnerability container_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "SAST") do
+            expect(dashboard).to have_vulnerability sast_scan_example_vuln
+          end
+
+          filter_report_and_perform(dashboard, "DAST") do
+            expect(dashboard).to have_vulnerability dast_scan_example_vuln
+          end
+        end
+      end
+
+      it 'displays the Dependency List' do
+        Page::Project::Menu.perform(&:click_on_dependency_list)
+
+        EE::Page::Project::Secure::DependencyList.perform do |dependency_list|
+          expect(dependency_list).to have_dependency_count_of number_of_dependencies_in_fixture
+        end
+      end
+    end
+
+    def filter_report_and_perform(page, report)
+      page.filter_report_type report
+      yield
+      page.filter_report_type report # Disable filter to avoid combining
+    end
+  end
+end

data/qa/specs/features/ee/browser_ui/secure/vulnerability_management_spec.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module QA
+  context 'Secure', :docker, :runner do
+    describe 'Security Reports in a Merge Request' do
+      let(:sast_vuln_count) { 5 }
+      let(:dependency_scan_vuln_count) { 4 }
+      let(:container_scan_vuln_count) { 8 }
+      let(:vuln_name) { "Regular Expression Denial of Service in debug" }
+      let(:remediable_vuln_name) { "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js" }
+
+      after(:all) do
+        @runner.remove_via_api!
+
+        Runtime::Feature.enable('job_log_json') if @job_log_json_flag_enabled
+      end
+
+      before(:all) do
+        @executor = "qa-runner-#{Time.now.to_i}"
+
+        Flow::Login.sign_in
+
+        @project = Resource::Project.fabricate_via_api! do |p|
+          p.name = Runtime::Env.auto_devops_project_name || 'project-with-secure'
+          p.description = 'Project with Secure'
+          p.auto_devops_enabled = false
+          p.initialize_with_readme = true
+        end
+
+        @runner = Resource::Runner.fabricate! do |runner|
+          runner.project = @project
+          runner.name = @executor
+          runner.tags = %w[qa test]
+        end
+
+        # Push fixture to generate Secure reports
+        @source = Resource::Repository::ProjectPush.fabricate! do |push|
+          push.project = @project
+          push.directory = Pathname
+            .new(__dir__)
+            .join('../../../../../ee/fixtures/secure_premade_reports')
+          push.commit_message = 'Create Secure compatible application to serve premade reports'
+          push.branch_name = 'secure-mr'
+        end
+
+        @merge_request = Resource::MergeRequest.fabricate_via_api! do |mr|
+          mr.project = @project
+          mr.source_branch = 'secure-mr'
+          mr.target_branch = 'master'
+          mr.source = @source
+          mr.target = 'master'
+          mr.target_new_branch = false
+        end
+
+        @project.visit!
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success)
+      end
+
+      before do
+        Flow::Login.sign_in_unless_signed_in
+        @merge_request.visit!
+      end
+
+      it 'can dismiss a vulnerability with a reason' do
+        dismiss_reason = "Vulnerability not applicable"
+
+        Page::MergeRequest::Show.perform do |merge_request|
+          expect(merge_request).to have_vulnerability_report
+          merge_request.dismiss_vulnerability_with_reason(vuln_name, dismiss_reason)
+          merge_request.click_vulnerability(vuln_name)
+
+          expect(merge_request).to have_opened_dismissed_vulnerability(dismiss_reason)
+        end
+      end
+
+      it 'can create an issue from a vulnerability' do
+        Page::MergeRequest::Show.perform do |merge_request|
+          expect(merge_request).to have_vulnerability_report
+          merge_request.create_vulnerability_issue(vuln_name)
+        end
+
+        Page::Project::Issue::Show.perform do |issue|
+          expect(issue).to have_title("Investigate vulnerability: #{vuln_name}")
+        end
+      end
+
+      it 'can create an auto-remediation MR' do
+        Page::MergeRequest::Show.perform do |merge_request|
+          expect(merge_request).to have_vulnerability_report
+          merge_request.resolve_vulnerability_with_mr remediable_vuln_name
+
+          # Context changes as resolve method creates new MR
+          expect(merge_request).to have_title remediable_vuln_name
+        end
+      end
+    end
+  end
+end

data/qa/specs/features/sanity/framework_spec.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module QA
+  context 'Framework sanity checks', :orchestrated, :framework do
+    describe 'Passing orchestrated example' do
+      it 'succeeds' do
+        Runtime::Browser.visit(:gitlab, Page::Main::Login)
+
+        expect(page).to have_text('A complete DevOps platform')
+      end
+    end
+
+    describe 'Failing orchestrated example' do
+      it 'fails' do
+        Runtime::Browser.visit(:gitlab, Page::Main::Login)
+
+        expect(page).to have_text("These Aren't the Texts You're Looking For", wait: 1)
+      end
+    end
+  end
+end

data/qa/specs/helpers/quarantine.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'rspec/core'
+
+module QA::Specs::Helpers
+  module Quarantine
+    include RSpec::Core::Pending
+
+    extend self
+
+    def configure_rspec
+      RSpec.configure do |config|
+        config.before(:context, :quarantine) do
+          Quarantine.skip_or_run_quarantined_contexts(config.inclusion_filter.rules, self.class)
+        end
+
+        config.before do |example|
+          Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example)
+        end
+      end
+    end
+
+    # Skip the entire context if a context is quarantined. This avoids running
+    # before blocks unnecessarily.
+    def skip_or_run_quarantined_contexts(filters, example)
+      return unless example.metadata.key?(:quarantine)
+
+      skip_or_run_quarantined_tests_or_contexts(filters, example)
+    end
+
+    # Skip tests in quarantine unless we explicitly focus on them.
+    def skip_or_run_quarantined_tests_or_contexts(filters, example)
+      if filters.key?(:quarantine)
+        included_filters = filters_other_than_quarantine(filters)
+
+        # If :quarantine is focused, skip the test/context unless its metadata
+        # includes quarantine and any other filters
+        # E.g., Suppose a test is tagged :smoke and :quarantine, and another is tagged
+        # :ldap and :quarantine. If we wanted to run just quarantined smoke tests
+        # using `--tag quarantine --tag smoke`, without this check we'd end up
+        # running that ldap test as well because of the :quarantine metadata.
+        # We could use an exclusion filter, but this way the test report will list
+        # the quarantined tests when they're not run so that we're aware of them
+        skip("Only running tests tagged with :quarantine and any of #{included_filters.keys}") if should_skip_when_focused?(example.metadata, included_filters)
+      else
+        if example.metadata.key?(:quarantine)
+          quarantine_message = %w(In quarantine)
+          quarantine_tag = example.metadata[:quarantine]
+
+          if !!quarantine_tag
+            quarantine_message << case quarantine_tag
+                                  when String
+                                    ": #{quarantine_tag}"
+                                  when Hash
+                                    ": #{quarantine_tag[:issue]}"
+                                  else
+                                    ''
+                                  end
+          end
+
+          skip(quarantine_message.join(' ').strip)
+        end
+      end
+    end
+
+    def filters_other_than_quarantine(filter)
+      filter.reject { |key, _| key == :quarantine }
+    end
+
+    # Checks if a test or context should be skipped.
+    #
+    # Returns true if
+    # - the metadata does not includes the :quarantine tag
+    # or if
+    # - the metadata includes the :quarantine tag
+    # - and the filter includes other tags that aren't in the metadata
+    def should_skip_when_focused?(metadata, included_filters)
+      return true unless metadata.key?(:quarantine)
+      return false if included_filters.empty?
+
+      (metadata.keys & included_filters.keys).empty?
+    end
+  end
+end

data/qa/specs/loop_runner.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module QA
+  module Specs
+    module LoopRunner
+      module_function
+
+      def run(args)
+        start = Time.now
+        loop_duration = 60 * QA::Runtime::Env.gitlab_qa_loop_runner_minutes
+
+        while Time.now - start < loop_duration
+          RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status|
+            abort if status.nonzero?
+          end
+          RSpec.clear_examples
+        end
+      end
+    end
+  end
+end

data/qa/specs/parallel_runner.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module QA
+  module Specs
+    module ParallelRunner
+      module_function
+
+      def run(args)
+        unless args.include?('--')
+          index = args.index { |opt| opt.include?('features') }
+
+          args.insert(index, '--') if index
+        end
+
+        env = {}
+        Runtime::Env::ENV_VARIABLES.each_key do |key|
+          env[key] = ENV[key] if ENV[key]
+        end
+        env['QA_RUNTIME_SCENARIO_ATTRIBUTES'] = Runtime::Scenario.attributes.to_json
+        env['GITLAB_QA_ACCESS_TOKEN'] = Runtime::API::Client.new(:gitlab).personal_access_token unless env['GITLAB_QA_ACCESS_TOKEN']
+
+        cmd = "bundle exec parallel_test -t rspec --combine-stderr --serialize-stdout -- #{args.flatten.join(' ')}"
+        ::Open3.popen2e(env, cmd) do |_, out, wait|
+          out.each { |line| puts line }
+
+          exit wait.value.exitstatus
+        end
+      end
+    end
+  end
+end

data/qa/specs/runner.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'knapsack'
+require 'rspec/core'
+require 'rspec/expectations'
+
+module QA
+  module Specs
+    class Runner < Scenario::Template
+      attr_accessor :tty, :tags, :options
+
+      DEFAULT_TEST_PATH_ARGS = ['--', File.expand_path('./features', __dir__)].freeze
+
+      def initialize
+        @tty = false
+        @tags = []
+        @options = []
+      end
+
+      def paths_from_knapsack
+        allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
+
+        QA::Runtime::Logger.info ''
+        QA::Runtime::Logger.info 'Report specs:'
+        QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
+        QA::Runtime::Logger.info ''
+        QA::Runtime::Logger.info 'Leftover specs:'
+        QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
+        QA::Runtime::Logger.info ''
+
+        ['--', allocator.node_tests]
+      end
+
+      def rspec_tags
+        tags_for_rspec = []
+
+        if tags.any?
+          tags.each { |tag| tags_for_rspec.push(['--tag', tag.to_s]) }
+        else
+          tags_for_rspec.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any?
+        end
+
+        tags_for_rspec.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled?
+
+        tags_for_rspec.push(%w[--tag ~skip_live_env]) if QA::Runtime::Env.dot_com?
+
+        QA::Runtime::Env.supported_features.each_key do |key|
+          tags_for_rspec.push(%W[--tag ~requires_#{key}]) unless QA::Runtime::Env.can_test? key
+        end
+
+        tags_for_rspec
+      end
+
+      def perform
+        args = []
+        args.push('--tty') if tty
+        args.push(rspec_tags)
+        args.push(options)
+
+        if Runtime::Env.knapsack?
+          args.push(paths_from_knapsack)
+        else
+          args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} }
+        end
+
+        if Runtime::Scenario.attributes[:parallel]
+          ParallelRunner.run(args.flatten)
+        elsif Runtime::Scenario.attributes[:loop]
+          LoopRunner.run(args.flatten)
+        else
+          RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status|
+            abort if status.nonzero?
+          end
+        end
+      end
+    end
+  end
+end