gitlab-secret_detection 0.1.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d72d76a2e018b5e645b7e3e2e7c4277ad1b6d464300a450ea118a8335bc3f962
-  data.tar.gz: e43b97d92391548323a25ae88a0814cc1ee9770373ea9646bb52982e9085d770
+  metadata.gz: ff032818993f54b85b958ad43293df4e9112ad20b9d68ab654abf74ebeb778a7
+  data.tar.gz: e0556ec3bc97e6973bbb6c8ba4e041897eccfbb37e23d4af16ff4fccd4850eb0
 SHA512:
-  metadata.gz: 29e37b4f36f013350f2ce86c3dba13c9af689e5091608b2c259ba64ff0d3cd8bf200f07d6ee381dc1effd15e2eb657a1a5e56c8d4de23163e2452149001b265d
-  data.tar.gz: 0ac708cee73ec8f5b3b32545c31481cffab6cc648e5963f50115d20495cca0d8ce8f07e29a9285da53b0e55f7ee30ad9a8fcfe9b2925f07dd16c0a337b336864
+  metadata.gz: ae963ac226ddab5d154879c078b44581f859232175ecf69269818a0e880368855469f10265a63d408089b5da7d56e37f51a865bbd9efc66ddd5474d1b718af0a
+  data.tar.gz: ff4c2bff936d88972a1e51563e2f6a0fb61f3c431213058fccae5e7abd18ac72636e37099a0dece9f083424795078bf2b69dd224aa88b27e6752bed1ae9d37ee

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright GitLab B.V.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,337 @@
+# Secret Detection Service
+
+Secret Detection service is primarily responsible for detecting secrets in the given input payloads with RPC methods as the communication interface served via gRPC.
+This service will initially be invoked by Rails monolith when performing access checks for Git Push event, and eventually extended for the other usecases too.
+
+Reference Issue: https://gitlab.com/groups/gitlab-org/-/epics/13792
+
+#### Tools and Framework
+
+- Ruby `3.2.5`
+- gRPC framework for serving RPC requests
+
+## Feature Distribution
+
+In addition to offering the feature as an gRPC service, this project also includes the provision for distributing the same feature into a Ruby Gem.
+This provision was added to fulfil [certain limitations](https://gitlab.com/gitlab-org/gitlab/-/issues/462359#note_1915874628). Here's the illustration representing
+the approach:
+
+![Feature Distribution](./doc/rpc_service.png "Feature Distribution")
+
+## Project Layout
+
+```
+├── .runway
+│   ├── runway.yml                       # Runway configuration file for Production environment
+│   ├── runway-staging.yml               # Runway configuration file for Staging environment
+│   └── env-*.yml                        # Environment vars for the respective environments. Uses vault for secrets
+├── bin
+│   └── start_server                     # gRPC server initiator that loads the server configuration file
+├── config
+│   └── log.rb                           # Logger configuration
+├── ci
+│   └── scripts/..                       # CI scripts used in the CI templates
+│   └── templates
+│       ├── build.yml                    # CI jobs for building container image and ruby gems
+│       ├── test.yml                     # CI jobs for running tests
+│       └── release.yml                  # CI jobs for releases related to Ruby gem, GitLab releases
+├── lib
+│   └── gitlab
+│       └── secret_detection
+│           ├── version.rb               # Secret Detection Gem release version
+│           ├── core/..                  # Secret detection logic (most of it pulled from existing gem)
+│           └── grpc
+│               ├── generated/..         # gRPC generated files and secret detection gRPC service
+│               ├── client/..            # gRPC client to invoke secret detection service's RPC endpoints 
+│               └── scanner_service.rb   # Secret detection gRPC service implementation
+├── examples
+│   └── sample-client/..                 # Sample Ruby RPC client that connects with gRPC server and calls RPC scan
+├── proto
+│   └── secret_detection.proto           # Service Definition file containing gRPC request and response interface
+├── server
+│   ├── interceptors/..                  # gRPC server-side interceptors like Auth, Log etc.
+│   └── server.rb                        # gRPC server file with configuration
+├── spec/..                              # Rspec tests and related test helpers
+├── gitlab-secret_detection.gemspec      # Gemspec file for Ruby Gem
+├── Dockerfile                           # Dockerfile for running gRPC server
+└── Makefile                             # All the CLI commands placed here
+```
+
+### Makefile commands
+
+Usage `make <command>`
+
+| Command             | Description                                                                                                                     |
+|---------------------|---------------------------------------------------------------------------------------------------------------------------------|
+| `install`           | Installs ruby gems in the project using Ruby bundler                                                                            |
+| `lint_fix`          | Fixes all the fixable Rubocop lint offenses                                                                                     |
+| `gem_clean`         | Cleans existing gem file(if any) generated through gem build process                                                            |
+| `gem_build`         | Builds Ruby gem file wrapping secret detection logic (lib directory)                                                            |
+| `generate_proto`    | Generates ruby(.rb) files for the Protobud Service Definition files(.proto)                                                     |
+| `grpc_docker_build` | Builds a docker container image for gRPC server                                                                                 |
+| `grpc_docker_serve` | Runs gRPC server via docker container listening on port 8080. Run `grpc_docker_build` make command before running this command. |
+| `grpc_serve`        | Runs gRPC server on the CLI listening on port 50001. Run `install` make command before running this command.                    |
+| `run_core_tests`    | Runs RSpec tests for Secret Detection core logic                                                                                |
+| `run_grpc_tests`    | Runs RSpec tests for Secret Detection gRPC endpoints                                                                            |
+| `run_all_tests`     | Runs all the RSpec tests in the project                                                                                         |
+
+## Generating a ruby gem
+
+In the project directory, run `make gem` command in the terminal that builds a ruby gem(ex: `secret_detection-0.1.0.gem`) in the root of
+the project directory.
+
+## Server
+
+This project currently runs on **gRPC** server.
+
+### Service Definitions
+
+- Scanner Service: [ProtoBuf Service Definition](proto/secret_detection.proto)
+
+- Health Check Service: [ProtoBuf Service Definition](https://github.com/grpc/grpc/blob/v1.64.x/doc/health-checking.md#service-definition)
+
+
+### Running the server locally
+
+Pre-requisite: gRPC installed on your system (`brew install grpc`)
+
+| server | mode   | command                                            | Listening port |
+|--------|--------|----------------------------------------------------|----------------|
+| gRPC   | CLI    | `make grpc_serve`                                  | `50001`        |
+| gRPC   | Docker | `make grpc_docker_build && make grpc_docker_serve` | `8080`         |
+
+gRPC server port can be configured via `RPC_SERVER_PORT` environment variable
+
+### Calling gRPC endpoints from terminal
+
+Pre-requisite:
+ - gRPC installed on your system (via `brew install grpc`)
+ - gRPC client like Postman or you can install `grpcurl` (via `brew install grpcurl`)
+
+**Authentication:**
+
+The RPC service uses a basic form of token-based authentication. When invoking an RPC request
+on the client side, we need to append an `x-sd-auth` RPC Header whose value is dependent
+on the environment where the server is running.
+
+The auth token value when the server is running in:
+
+- Localhost: `12345`
+- Staging: `env/staging/service/secret-detection/API_AUTH_TOKEN` path in [Vault](https://vault.gitlab.net/)
+- Production: `env/production/service/secret-detection/API_AUTH_TOKEN` path in [Vault](https://vault.gitlab.net/)
+
+Note that only Secret Detection RPC requests are guarded with authentication.
+
+More details can be found in this [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/467531?work_item_iid=477700).
+
+#### Health Check
+
+**RPC Method:** `grpc.health.v1.Health/Check`
+
+<details><summary>Example</summary>
+
+```shell
+$ grpcurl -plaintext -d '{"service":"gitlab.secret_detection.Scanner"}' localhost:50001 grpc.health.v1.Health/Check
+```
+
+You should see the following response as a result:
+
+```shell
+{
+  "status": "SERVING"
+}
+```
+
+</details>
+
+#### Secret Detection Scan (Unary RPC Call)
+
+**RPC Method:** `gitlab.secret_detection.Scanner/Scan`
+**Default Timeout(configurable):** per-request: `180 seconds`, per-payload: `30 seconds`
+
+
+<details><summary>Example: Basic</summary>
+
+
+```shell
+$ grpcurl -d @ \
+  localhost:50001 \
+  -rpc-header 'x-sd-auth:12345' \
+  gitlab.secret_detection.Scanner/Scan <<EOM
+{
+  "payloads": [
+    {
+      "id": "94283",
+      "data": "glpat-12345123451234512345"
+    }
+  ]
+}
+EOM
+```
+
+You should see the following response as a result:
+
+```json
+{
+  "results": [
+    {
+      "payload_id": "94283",
+      "status": "FOUND",
+      "type": "gitlab_personal_access_token",
+      "description": "GitLab Personal Access Token",
+      "line_number": 1
+    }
+  ],
+  "status": "FOUND"
+}
+```
+
+</details>
+
+
+<details><summary>Example: Using Exclusions and Tags</summary>
+
+```shell
+$ grpcurl -d @ \
+  localhost:50001 \
+  -rpc-header 'x-sd-auth:12345' \
+  gitlab.secret_detection.Scanner/Scan <<EOM
+{
+    "payloads": [
+        {
+            "id": "94283",
+            "data": "glpat-12345123451234512345"
+        },
+        {
+            "id": "37405",
+            "data": "glrt-12345123451234512345"
+        },
+        {
+            "id": "14954",
+            "data": "GR1348941__DUMMY_RUNNER_TOKEN"
+        }
+    ],
+    "exclusions": [
+        {
+            "exclusion_type": "EXCLUSION_TYPE_RULE",
+            "value": "gitlab_personal_access_token"
+        },
+        {
+            "exclusion_type": "EXCLUSION_TYPE_RAW_VALUE",
+            "value": "glrt-12345123451234512345"
+        }
+    ],
+    "tags": [
+        "gitlab_blocking"
+    ]
+}
+EOM
+```
+
+You should see the following response as a result:
+
+```json
+{
+  "results": [
+    {
+      "payload_id": "14954",
+      "status": "STATUS_FOUND",
+      "type": "gitlab_pipeline_trigger_token",
+      "description": "GitLab Pipeline Trigger Token",
+      "line_number": 1
+    }
+  ],
+  "status": "STATUS_FOUND"
+}
+```
+
+</details>
+
+
+#### Secret Detection Scan (Bi-directional RPC Streaming)
+
+**RPC Method:** `gitlab.secret_detection.Scanner/ScanStream`
+**Default Timeout(configurable):** per-request: `180 seconds`, per-payload: `30 seconds`
+
+Bi-directional RPC streaming allows the server to receive a continuous stream of requests and respond with a
+corresponding stream of responses as each request is processed. Unlike unary RPC calls, where requests are sent once and
+the channel closes after a single response, bi-directional streaming keeps the RPC channel open to handle ongoing
+requests and responses, enabling continuous communication between the client and server.
+
+
+<details><summary>Example</summary>
+
+To try this out, we will accept the request input from STDIN. As on when you provide request input json in the terminal,
+you should see a corresponding server response. The connection will continue to remain open for accepting
+requests unless you explicitly close the connection with `Ctrl+C`.
+
+Here's the command to start an RPC streaming channel on the terminal:
+
+```shell
+$ grpcurl -d @ localhost:50001 -rpc-header 'x-sd-auth:12345' gitlab.secret_detection.Scanner/ScanStream
+```
+
+Once the channel is open, enter the following sample request input in the terminal:
+
+```json
+{
+  "payloads": [
+    {
+      "id": "94283",
+      "data": "glpat-12345123451234512345"
+    }
+  ]
+}
+```
+
+you should immediately see the Server's response like below:
+
+```json
+{
+  "results": [
+    {
+      "payload_id": "94283",
+      "status": "STATUS_FOUND",
+      "type": "gitlab_personal_access_token",
+      "description": "GitLab Personal Access Token",
+      "line_number": 1
+    }
+  ],
+  "status": "STATUS_FOUND"
+}
+```
+
+You may continue to provide more request inputs to the opened RPC channel to receive corresponding server responses. Features like Exclusions, Tag Filtering works same as outlined under Unary call section.
+
+</details>
+
+**NOTE**:
+- In case you're running the local server via Docker, replace `localhost:50001` with `localhost:8080` in the example `grpcurl` commands.
+- In case you want to access **staging** server, replace `localhost:50001` with `secret-detection.staging.runway.gitlab.net:443`
+
+
+## How to invoke the server from the code (as an RPC Client)
+
+There is a sample [Ruby RPC Client](examples/sample-client/sample_client.rb) in the project that contains following code for reference:
+ - Setting up the connection with the server (both local and remote server with SSL)
+ - Invoke Secret Detection Unary call
+ - Invoke Secret Detection bi-directional stream call
+
+Run `ruby examples/sample-client/sample_client.rb` on your terminal to run the sample RPC client calling both unary and streaming RPC Scan methods.
+
+## Benchmark
+
+RPC service is benchmarked using [`ghz`](https://ghz.sh), a powerful CLI-based tool for load testing and benchmarking gRPC services. More details added [here](https://gitlab.com/gitlab-org/gitlab/-/work_items/468107).
+
+## Project Status
+
+Secret Detection service's status can be tracked here: https://gitlab.com/gitlab-org/gitlab/-/issues/467531
+
+#### Changes made in the secret detection logic that were previously not present in the Gem
+
+- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
+the `initialize()` method now accepts parsed rules instead of ruleset file path
+- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
+gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
+- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
+method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.

data/config/log.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'grpc'
+require 'logger'
+
+# SD_ENV env var is used to determine which environment the
+# server is running. This var is defined in `.runway/env-<env>.yml` files.
+def local_env?
+  ENV.fetch('SD_ENV', 'localhost') == 'localhost'
+end
+
+module SDLogger
+  LOGGER = Logger.new $stderr, level: local_env? ? Logger::DEBUG : Logger::INFO
+
+  def logger
+    LOGGER
+  end
+end
+
+# Configure logger for GRPC
+module GRPC
+  extend SDLogger
+end

data/lib/gitlab/secret_detection/core/finding.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SecretDetection
+    module Core
+      # Finding is a data object representing a secret finding identified within a payload
+      class Finding
+        attr_reader :payload_id, :status, :line_number, :type, :description
+
+        def initialize(payload_id, status, line_number = nil, type = nil, description = nil)
+          @payload_id = payload_id
+          @status = status
+          @line_number = line_number
+          @type = type
+          @description = description
+        end
+
+        def ==(other)
+          self.class == other.class && other.state == state
+        end
+
+        def to_h
+          {
+            payload_id:,
+            status:,
+            line_number:,
+            type:,
+            description:
+          }
+        end
+
+        protected
+
+        def state
+          [payload_id, status, line_number, type, description]
+        end
+      end
+    end
+  end
+end