gitlab-secret_detection 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7ff1ed2f6fa1d52463144cfacf4a23bd191822d6660e5f4d3c15cefb349b9dd
-  data.tar.gz: 13af55c3efd41733108968de3d0c8d03648c726947170636a293db174521a81e
+  metadata.gz: 648b6d5277ac8e7948762533af39dc2f4ff0ae4c62fbdcc6d5d32615a44ab815
+  data.tar.gz: afd2a580cb0a73bb84a401616ec622e1e14d2b75021036a959f933ed75864cf6
 SHA512:
-  metadata.gz: 6698802604dffeff97940812c8463dbb21c73698c8f438e825b482d5dc68c97953aaa983d03af7de26f37ff9ec7d1e1418966e495c2ef8f308eb05f40d4ae601
-  data.tar.gz: b6e1b308dddfa644500184058228faf67458702c82994fc9c475ce03ef47a51323f309c1163287a2d1ce0bbd13aaec2a19c71e4a8ca711f6a6beaefbda2649d1
+  metadata.gz: 7c49b71891f6e13d8dc252936f18df838d8e9d42000cbc1f4f3d7fe56b258a7e2c17bce7a7f0e771bc16991fa46bacf641a636a1d1efd3ef23de6b0a31a011b8
+  data.tar.gz: af93c82a0025be12bc82fe6ff62b290fe193a2338ae2f98d83bc2729d1c7cc35a223330a3a9907a9f715c009127e93ba2d13fc33982d2f13a179d8548f5cc36e

data/lib/gitlab/secret_detection/core/scanner.rb

@@ -4,6 +4,7 @@ require 're2'
 require 'logger'
 require 'timeout'
 require 'English'
+require 'parallel'
 
 module GitLab
   module SecretDetection
@@ -24,6 +25,14 @@ module GitLab
         DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
         # Tags used for creating default pattern matcher
         DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
+        # Max no of child processes to spawn per request
+        # ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160
+        MAX_PROCS_PER_REQUEST = 5
+        # Minimum cumulative size of the payloads required to spawn and
+        # run the scan within a new subprocess.
+        MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB
+        # Whether to run scan in subprocesses or not. Default is false.
+        RUN_IN_SUBPROCESS = false
 
         # Initializes the instance with logger along with following operations:
         # 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
@@ -58,6 +67,13 @@ module GitLab
         #           For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
         #           [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
         #
+        # NOTE:
+        # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
+        # offloading regex operations on large payloads to sub-processes. However, it does not assure the improvement
+        # in the overall latency of the scan, specifically in the case of smaller payloads, where the overhead of
+        # forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based
+        # execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160.
+        #
         # Returns an instance of GitLab::SecretDetection::Core::Response by following below structure:
         # {
         #     status: One of the Core::Status values
@@ -70,7 +86,8 @@ module GitLab
           payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
           raw_value_exclusions: [],
           rule_exclusions: [],
-          tags: DEFAULT_PATTERN_MATCHER_TAGS
+          tags: DEFAULT_PATTERN_MATCHER_TAGS,
+          subprocess: RUN_IN_SUBPROCESS
         )
 
           return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
@@ -87,11 +104,13 @@ module GitLab
 
             next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
 
-            secrets = run_scan(
+            scan_args = {
               payloads: matched_payloads, payload_timeout:,
               pattern_matcher: build_pattern_matcher(tags:),
               raw_value_exclusions:, rule_exclusions:
-            )
+            }
+
+            secrets = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
 
             scan_status = overall_scan_status(secrets)
 
@@ -205,6 +224,36 @@ module GitLab
           end
         end
 
+        def run_scan_within_subprocess(
+          payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [],
+          rule_exclusions: [])
+          payload_sizes = payloads.map(&:size)
+          grouped_payload_indices = group_by_chunk_size(payload_sizes)
+
+          grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
+
+          found_secrets = Parallel.flat_map(
+            grouped_payloads,
+            in_processes: MAX_PROCS_PER_REQUEST,
+            isolation: true # do not reuse sub-processes
+          ) do |grouped_payload|
+            grouped_payload.flat_map do |payload|
+              Timeout.timeout(payload_timeout) do
+                find_secrets_in_payload(
+                  payload:,
+                  pattern_matcher:,
+                  raw_value_exclusions:, rule_exclusions:
+                )
+              end
+            rescue Timeout::Error => e
+              logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
+              Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
+            end
+          end
+
+          found_secrets.freeze
+        end
+
         # Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
         # literal values to exclude from the input before the scan, also SD rules to exclude during
         # the scan.
@@ -268,6 +317,35 @@ module GitLab
             Core::Status::FOUND_WITH_ERRORS
           end
         end
+
+        # This method accepts an array of payload sizes(in bytes) and groups them into an array
+        # of arrays structure where each element is the group of indices of the input
+        # array whose cumulative payload sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+
+        def group_by_chunk_size(payload_size_arr)
+          cumulative_size = 0
+          chunk_indexes = []
+          chunk_idx_start = 0
+
+          payload_size_arr.each_with_index do |size, index|
+            cumulative_size += size
+            next unless cumulative_size >= MIN_CHUNK_SIZE_PER_PROC_BYTES
+
+            chunk_indexes << (chunk_idx_start..index).to_a
+
+            chunk_idx_start = index + 1
+            cumulative_size = 0
+          end
+
+          if cumulative_size.positive? && (chunk_idx_start < payload_size_arr.length)
+            chunk_indexes << if chunk_idx_start == payload_size_arr.length - 1
+                               [chunk_idx_start]
+                             else
+                               (chunk_idx_start..payload_size_arr.length - 1).to_a
+                             end
+          end
+
+          chunk_indexes
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-secret_detection
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - group::secret detection
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-04 00:00:00.000000000 Z
+date: 2024-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -40,6 +40,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.63.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: re2
   requirement: !ruby/object:Gem::Requirement