gitlab-secret_detection 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23eb987781013adea0ee3eaf3a52f3b7c89e09268a7dfff504d0346efbfeb8e2
-  data.tar.gz: 9a301fcbdb3b9a2bb76e4dd978a8d574a654a2a65ed4ec3c4f3fd962dfe67c3d
+  metadata.gz: f72d7d8262749305fdf1811df74637af4f7ba4bbf693fc41d2cdb34d1ca25238
+  data.tar.gz: 6896b4eb32dc734640c0dc125dd75e79b765f27aa618f73fec81c83c0c20df93
 SHA512:
-  metadata.gz: 9f2ff05eb818ffeed0a1d1736d59de48822dd8f2d2fb8f56526dd6aa2e36c901eb1592a8372f0999c72c2f8ee6e8fc0b8ac7bf37021f10e2f405aebb2b890a5b
-  data.tar.gz: 38cdab69b370d7f82784d7e861784cda7a6867341bab08f238ee54aa427c61b5056fced813ab9c9161f8a5b14d8493cc6be29e10779187f76b61980586757b23
+  metadata.gz: 2d35dfa136b9524f851a6ed20885aa6d50b9334f12166fd9d131ff523ed38bed1264bad953fd6fa58e63fe2c5f9c62be129d3be231258eef25990803b9b67d4e
+  data.tar.gz: 331a74ba7a779f275717c769c0f63785d7e74d25b07da5d07f44c08a1a0c7f343eec3ba7fd8285d549d298fac4d93bdc3f2bc122ea7cb30bd567e4c75e43efdb

data/lib/gitlab/secret_detection/core/response.rb

@@ -7,14 +7,16 @@ module Gitlab
       #
       # +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
       # +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
-      # +metadata+:: Hash object containing additional meta information about the response. It is currently used
       #   to embed more information on error.
+      # +applied_exclusions+:: Array of Exclusions that were applied during this scan.
+      # +metadata+:: Hash object containing additional meta information about the response. It is currently used
       class Response
-        attr_reader :status, :results, :metadata
+        attr_reader :status, :results, :applied_exclusions, :metadata
 
-        def initialize(status, results = [], metadata = {})
+        def initialize(status:, results: [], applied_exclusions: [], metadata: {})
           @status = status
           @results = results
+          @applied_exclusions = applied_exclusions
           @metadata = metadata
         end
 
@@ -25,15 +27,21 @@ module Gitlab
         def to_h
           {
             status:,
-            metadata:,
-            results: results&.map(&:to_h)
+            results: results&.map(&:to_h),
+            applied_exclusions:,
+            metadata:
           }
         end
 
         protected
 
         def state
-          [status, metadata, results]
+          [
+            status,
+            results,
+            applied_exclusions,
+            metadata
+          ]
         end
       end
     end

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -90,7 +90,7 @@ module Gitlab
           subprocess: RUN_IN_SUBPROCESS
         )
 
-          return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
+          return Core::Response.new(status: Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
 
           # assign defaults since grpc passing zero timeout value to `Timeout.timeout(..)` makes it effectively useless.
           timeout = DEFAULT_SCAN_TIMEOUT_SECS unless timeout.positive?
@@ -102,24 +102,26 @@ module Gitlab
 
             matched_payloads = filter_by_keywords(keyword_matcher, payloads)
 
-            next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
+            next Core::Response.new(status: Core::Status::NOT_FOUND) if matched_payloads.empty?
 
             scan_args = {
-              payloads: matched_payloads, payload_timeout:,
+              payloads: matched_payloads,
+              payload_timeout:,
               pattern_matcher: build_pattern_matcher(tags:),
-              raw_value_exclusions:, rule_exclusions:
+              raw_value_exclusions:,
+              rule_exclusions:
             }
 
-            secrets = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
+            secrets, applied_exclusions = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
 
             scan_status = overall_scan_status(secrets)
 
-            Core::Response.new(scan_status, secrets)
+            Core::Response.new(status: scan_status, results: secrets, applied_exclusions:)
           end
         rescue Timeout::Error => e
           logger.error "Secret detection operation timed out: #{e}"
 
-          Core::Response.new(Core::Status::SCAN_TIMEOUT)
+          Core::Response.new(status: Core::Status::SCAN_TIMEOUT)
         end
 
         private
@@ -208,25 +210,42 @@ module Gitlab
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan when performed on the payloads.
         def run_scan(
-          payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [], rule_exclusions: [])
-          payloads.flat_map do |payload|
+          payloads:,
+          payload_timeout:,
+          pattern_matcher:,
+          raw_value_exclusions: [],
+          rule_exclusions: []
+        )
+          all_applied_exclusions = Set.new
+
+          all_findings = payloads.flat_map do |payload|
             Timeout.timeout(payload_timeout) do
-              find_secrets_in_payload(
+              findings, applied_exclusions = find_secrets_in_payload(
                 payload:,
                 pattern_matcher:,
-                raw_value_exclusions:, rule_exclusions:
+                raw_value_exclusions:,
+                rule_exclusions:
               )
+              all_applied_exclusions.merge(applied_exclusions)
+              findings
             end
           rescue Timeout::Error => e
             logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+
             Core::Finding.new(payload.id,
               Core::Status::PAYLOAD_TIMEOUT)
           end
+          [all_findings.freeze, all_applied_exclusions.to_a.freeze]
         end
 
         def run_scan_within_subprocess(
-          payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [],
-          rule_exclusions: [])
+          payloads:,
+          payload_timeout:,
+          pattern_matcher:,
+          raw_value_exclusions: [],
+          rule_exclusions: []
+        )
+          all_applied_exclusions = Set.new
           payload_sizes = payloads.map(&:size)
           grouped_payload_indices = group_by_chunk_size(payload_sizes)
 
@@ -239,19 +258,22 @@ module Gitlab
           ) do |grouped_payload|
             grouped_payload.flat_map do |payload|
               Timeout.timeout(payload_timeout) do
-                find_secrets_in_payload(
+                findings, applied_exclusions = find_secrets_in_payload(
                   payload:,
                   pattern_matcher:,
                   raw_value_exclusions:, rule_exclusions:
                 )
+                all_applied_exclusions.merge(applied_exclusions)
+                findings
               end
             rescue Timeout::Error => e
               logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+
               Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
             end
           end
 
-          found_secrets.freeze
+          [found_secrets.freeze, all_applied_exclusions.to_a.freeze]
         end
 
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
@@ -259,6 +281,7 @@ module Gitlab
         # the scan.
         def find_secrets_in_payload(payload:, pattern_matcher:, raw_value_exclusions: [], rule_exclusions: [])
           findings = []
+          applied_exclusions = Set.new
 
           payload_offset = payload.respond_to?(:offset) ? payload.offset : 0
 
@@ -268,6 +291,7 @@ module Gitlab
             unless raw_value_exclusions.empty?
               raw_value_exclusions.each do |value|
                 line.gsub!(value, '') # replace input that doesn't contain allowed value in it
+                applied_exclusions << value # TODO we need the id of the exclusion
               end
             end
 
@@ -284,19 +308,30 @@ module Gitlab
             matches.each do |match_idx|
               rule = rules[match_idx]
 
-              next if rule_exclusions.include?(rule[:id])
+              next if applied_rule_exclusion?(rule[:id], rule_exclusions, applied_exclusions)
 
               title = rule[:title].nil? ? rule[:description] : rule[:title]
-              findings << Core::Finding.new(payload.id, Core::Status::FOUND, line_no, rule[:id],
-                title)
+
+              findings << Core::Finding.new(
+                payload.id,
+                Core::Status::FOUND,
+                line_no,
+                rule[:id],
+                title
+              )
             end
           end
 
-          findings.freeze
+          [findings.freeze, applied_exclusions]
         rescue StandardError => e
           logger.error "Secret Detection scan failed on the payload(id:#{payload.id}): #{e}"
 
-          Core::Finding.new(payload.id, Core::Status::SCAN_ERROR)
+          [[Core::Finding.new(payload.id, Core::Status::SCAN_ERROR)], []]
+        end
+
+        def applied_rule_exclusion?(type, rule_exclusions, applied_exclusions)
+          applied_exclusion = rule_exclusions&.find { |rule_exclusion| rule_exclusion == type }
+          applied_exclusion && (applied_exclusions << applied_exclusion)
         end
 
         # Validates the given payloads by verifying the type and

data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb

@@ -17,10 +17,11 @@ module Gitlab
         # Time to wait for the response from the service
         REQUEST_TIMEOUT_SECONDS = 10 # 10 seconds
 
-        def initialize(host, secure: false, compression: true)
+        def initialize(host, secure: false, compression: true, logger: nil)
           @host = host
           @secure = secure
           @compression = compression
+          @logger = logger.nil? ? LOGGER : logger
         end
 
         # Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
@@ -116,14 +117,18 @@ module Gitlab
         def with_rescued_errors
           yield
         rescue ::GRPC::Unauthenticated
-          SecretDetection::Core::Response.new(SecretDetection::Core::Status::AUTH_ERROR)
+          SecretDetection::Core::Response.new(status: SecretDetection::Core::Status::AUTH_ERROR)
         rescue ::GRPC::InvalidArgument => e
           SecretDetection::Core::Response.new(
-            SecretDetection::Core::Status::INPUT_ERROR, nil, { message: e.details, **e.metadata }
+            status: SecretDetection::Core::Status::INPUT_ERROR,
+            results: nil,
+            metadata: { message: e.details, **e.metadata }
           )
         rescue ::GRPC::Unknown, ::GRPC::BadStatus => e
           SecretDetection::Core::Response.new(
-            SecretDetection::Core::Status::SCAN_ERROR, nil, { message: e.details }
+            status: SecretDetection::Core::Status::SCAN_ERROR,
+            results: nil,
+            metadata: { message: e.details }
           )
         end
 
@@ -131,13 +136,13 @@ module Gitlab
           response = grpc_response.to_h
 
           SecretDetection::Core::Response.new(
-            response[:status],
-            response[:results],
-            response[:metadata]
+            status: response[:status],
+            results: response[:results],
+            metadata: response[:metadata]
           )
         rescue StandardError => e
-          logger.error("Failed to convert to core response: #{e}")
-          SecretDetection::Core::Response.new(SecretDetection::Core::Status::SCAN_ERROR)
+          @logger.error("Failed to convert to core response: #{e}")
+          SecretDetection::Core::Response.new(status: SecretDetection::Core::Status::SCAN_ERROR)
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - group::secret detection