gitlab-secret_detection 0.11.1 → 0.39.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a749972fd361e8f84074f6b292f0a244ad5f86dd9d78b046cdd5593fed8ead8e
-  data.tar.gz: bdbc88f40aa8061577bcc4c618e3fc1691ef3b62276509300f8f2ee5be4c3e52
+  metadata.gz: 73f035d35d4ceca23cc77190ee6acb59e8c26e4a473102a9a56da16d3dc23986
+  data.tar.gz: c6b85135c9f0bd83268bab359efc0b69f33454c371e397fa77be0e5aea5efcb5
 SHA512:
-  metadata.gz: 72af511eae50a715a3c5023302ba3eafe7f7dd25af4a420bcc09b797c36fb924d2d70d074f5877bd5e21527347785b743d90b58e779dd90b43e16926f30351ed
-  data.tar.gz: 700627072f35b21367e561343f87dce664bf30e6b627194e56a3fbbe421bbd73794cf4e0f1fc3fae69382da92a4ac1067e2f5a842b51b777a9e8ce490cac49f3
+  metadata.gz: 4b127be2b764b05ae628dadfd0de811fa8d6e893b7d843ce2e60f4fc0af2f247580028350fa970fee5cc6980b71f2882bcc842fb6d5cd50ac3995276112714c6
+  data.tar.gz: 4961476f2092a893b11d82c357ea0e2138ddb07490edab4c207f19157f26ec44060f16e744739554bf0e0365cca38fda2813fd7c53dd01c7d8f9956270722e3a

data/README.md

@@ -7,7 +7,7 @@ Reference Issue: https://gitlab.com/groups/gitlab-org/-/epics/13792
 
 #### Tools and Framework
 
-- Ruby `3.2.5`
+- Ruby `3.3.X`
 - gRPC framework for serving RPC requests
 
 ## Feature Distribution
@@ -62,20 +62,21 @@ the approach:
 
 Usage `make <command>`
 
-| Command             | Description                                                                                                                     |
-|---------------------|---------------------------------------------------------------------------------------------------------------------------------|
-| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                 |
-| `install`           | Installs ruby gems in the project using Ruby bundler                                                                            |
-| `lint_fix`          | Fixes all the fixable Rubocop lint offenses                                                                                     |
-| `gem_clean`         | Cleans existing gem file(if any) generated through gem build process                                                            |
-| `gem_build`         | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
-| `generate_proto`    | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
-| `grpc_docker_build` | Builds a docker container image for gRPC server                                                                                 |
-| `grpc_docker_serve` | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
-| `grpc_serve`        | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
-| `run_core_tests`    | Runs RSpec tests for Secret Detection core logic                                                                                |
-| `run_grpc_tests`    | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
-| `run_all_tests`     | Runs all the RSpec tests in the project                                                                                         |
+| Command                          | Description                                                                                                                     |
+|----------------------------------|---------------------------------------------------------------------------------------------------------------------------------|
+| `install_secret_detection_rules` | Downloads secret-detection-rules based on package version defined in RULES_VERSION                                              |
+| `install`                        | Installs ruby gems in the project using Ruby bundler                                                                            |
+| `lint_fix`                       | Fixes all the fixable Rubocop lint offenses                                                                                     |
+| `gem_clean`                      | Cleans existing gem file(if any) generated through gem build process                                                            |
+| `gem_build`                      | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
+| `generate_proto`                 | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
+| `grpc_docker_build`              | Builds a docker container image for gRPC server                                                                                 |
+| `grpc_docker_serve`              | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
+| `grpc_serve`                     | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
+| `run_core_tests`                 | Runs RSpec tests for Secret Detection core logic                                                                                |
+| `run_grpc_tests`                 | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
+| `run_utils_tests`                | Runs RSpec tests for Secret Detection utilities                                                                                 |
+| `run_all_tests`                  | Runs all the RSpec tests in the project                                                                                         |
 
 
 ## Secret Detection Rules
@@ -166,9 +167,9 @@ You should see the following response as a result:
 
 
 ```shell
-$ grpcurl -d @ \
-  localhost:50001 \
+grpcurl -plaintext -d @ \
   -rpc-header 'x-sd-auth:12345' \
+  localhost:50001 \
   gitlab.secret_detection.Scanner/Scan <<EOM
 {
   "payloads": [
@@ -335,15 +336,22 @@ Run `ruby examples/sample-client/sample_client.rb` on your terminal to run the s
 
 RPC service is benchmarked using [`ghz`](https://ghz.sh), a powerful CLI-based tool for load testing and benchmarking gRPC services. More details added [here](https://gitlab.com/gitlab-org/gitlab/-/work_items/468107).
 
-## Project Status
+## Release Process
+
+We do three primary actions for every merge to `main` branch:
+
+- **Build and Publish SD ruby gem to RubyGems.org**:
+  - The latest version for releasing Secret Detection gem is pulled from `Gitlab::SecretDetection::Gem::VERSION` (located at`lib/gitlab/secret_detection/version.rb`).
+  - We build a ruby gem for the code snapshot and tag it to the extract release version.
+  - The script for publising the gem to RubyGems.org is available [here](ci/scripts/publish_ruby_gem.sh).
 
-Secret Detection service's status can be tracked here: https://gitlab.com/gitlab-org/gitlab/-/issues/467531
+- **Deploy SD gRPC server to GCP using Runway**:
+  - We build a docker container for the current code snapshot and tag it under `$CI_REGISTRY_IMAGE/image:$CI_COMMIT_SHORT_SHA` container registry path.
+  - The same container registry path is given as input to the Runway CI downstream which takes it forward for deploying in Staging and Production environments.
 
-#### Changes made in the secret detection logic that were previously not present in the Gem
+- **Make a GitLab Release**:
+  - We use a modified version of [`upsert git tag`](https://gitlab.com/gitlab-org/security-products/ci-templates/-/blob/master/includes-dev/upsert-git-tag.ym) job where instead of fetching the version from the first changelog entry, we fetch it from `Gitlab::SecretDetection::Gem::VERSION`. The rest of the behaviour is retained i.e., creating a tag from the version and then creating a new GitLab release against that tag.
+  - The job pulls the description of the latest version entry from the [`CHANGELOG.md`](CHANGELOG.md) and uses it for the Release description.
+  - The script for creating a git tag and making GitLab release is available [here](ci/scripts/make_gitlab_release.sh).
 
-- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
-the `initialize()` method now accepts parsed rules instead of ruleset file path
-- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
-gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
-- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
-method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
+*NOTE: There is no logical requirement for the versions defined in `Gitlab::SecretDetection::Gem::VERSION` and latest entry of `CHANGELOG.md` to be the same. However, we expect them to be the same to keep it consistent. We've added a CI job([`validate version sync`](ci/templates/validate.yml)) that ensures the version sync between them.*

data/lib/gitlab/secret_detection/core/response.rb

@@ -7,14 +7,18 @@ module Gitlab
       #
       # +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
       # +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
-      # +metadata+:: Hash object containing additional meta information about the response. It is currently used
       #   to embed more information on error.
+      # +applied_exclusions+:: Array of exclusions that were applied during this scan.
+      #   These can be either GRPC::Exclusions when used as a service, or `Security::ProjectSecurityExclusion
+      #   object when used as a gem.
+      # +metadata+:: Hash object containing additional meta information about the response. It is currently used
       class Response
-        attr_reader :status, :results, :metadata
+        attr_reader :status, :results, :applied_exclusions, :metadata
 
-        def initialize(status, results = [], metadata = {})
+        def initialize(status:, results: [], applied_exclusions: [], metadata: {})
           @status = status
           @results = results
+          @applied_exclusions = applied_exclusions
           @metadata = metadata
         end
 
@@ -25,15 +29,21 @@ module Gitlab
         def to_h
           {
             status:,
-            metadata:,
-            results: results&.map(&:to_h)
+            results: results&.map(&:to_h),
+            applied_exclusions:,
+            metadata:
           }
         end
 
         protected
 
         def state
-          [status, metadata, results]
+          [
+            status,
+            results,
+            applied_exclusions,
+            metadata
+          ]
         end
       end
     end

data/lib/gitlab/secret_detection/core/ruleset.rb

@@ -7,6 +7,14 @@ module Gitlab
   module SecretDetection
     module Core
       class Ruleset
+        # RulesetParseError is thrown when the code fails to parse the
+        # ruleset file from the given path
+        RulesetParseError = Class.new(StandardError)
+
+        # RulesetCompilationError is thrown when the code fails to compile
+        # the predefined rulesets
+        RulesetCompilationError = Class.new(StandardError)
+
         # file path where the secrets ruleset file is located
         RULESET_FILE_PATH = File.expand_path('secret_push_protection_rules.toml', __dir__)
 
@@ -21,18 +29,37 @@ module Gitlab
           @rule_data = parse_ruleset
         end
 
+        def extract_ruleset_version
+          @ruleset_version ||= if File.readable?(RULESET_FILE_PATH)
+                                 first_line = File.open(RULESET_FILE_PATH, &:gets)
+                                 first_line&.split(":")&.[](1)&.strip
+                               end
+        rescue StandardError => e
+          logger.error(message: "Failed to extract Secret Detection Ruleset version from ruleset file: #{e.message}")
+        end
+
         private
 
         attr_reader :path, :logger
 
         # parses given ruleset file and returns the parsed rules
         def parse_ruleset
-          # rule_file_content = File.read(path)
+          logger.info(
+            message: "Parsing local ruleset file",
+            ruleset_path: RULESET_FILE_PATH
+          )
           rules_data = TomlRB.load_file(path, symbolize_keys: true).freeze
+          ruleset_version = extract_ruleset_version
+
+          logger.info(
+            message: "Ruleset details fetched for running Secret Detection scan",
+            total_rules: rules_data[:rules]&.length,
+            ruleset_version:
+          )
           rules_data[:rules].freeze
         rescue StandardError => e
-          logger.error "Failed to parse secret detection ruleset from '#{path}' path: #{e}"
-          raise Core::Scanner::RulesetParseError
+          logger.error(message: "Failed to parse local secret detection ruleset: #{e.message}")
+          raise RulesetParseError, e
         end
       end
     end