gitlab-secret_detection 0.3.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a24f98710dabdb6384cc73105322372c12125c5caccddec60928dde4a8d86e39
|
4
|
+
data.tar.gz: 6f543facba201e091e553832bbae3d1f296547cea4558146077b15cac9f744d1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e06f9608d8bbb4f9e026eba705668a957215318a8e046a5dfdc97c87ce5569e7357c23040218b1122a459d9939e54fa162d7f7f08528cf99d82ede677b00e4f7
|
7
|
+
data.tar.gz: be989d9f5be6dd86ef752c4eb28b0738c157a844bb1bdb85b520a5f3060702df014090af03776659bb384469c5586b1e697ad1d52189f90c5e9c98afad65eae5
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Secret Detection Service
|
2
2
|
|
3
|
-
Secret Detection service is primarily responsible for detecting secrets in the given input payloads with RPC methods as the communication interface served via gRPC.
|
3
|
+
Secret Detection service is primarily responsible for detecting secrets in the given input payloads with RPC methods as the communication interface served via gRPC.
|
4
4
|
This service will initially be invoked by Rails monolith when performing access checks for Git Push event, and eventually extended for the other usecases too.
|
5
5
|
|
6
6
|
Reference Issue: https://gitlab.com/groups/gitlab-org/-/epics/13792
|
@@ -12,7 +12,7 @@ Reference Issue: https://gitlab.com/groups/gitlab-org/-/epics/13792
|
|
12
12
|
|
13
13
|
## Feature Distribution
|
14
14
|
|
15
|
-
In addition to offering the feature as an gRPC service, this project also includes the provision for distributing the same feature into a Ruby Gem.
|
15
|
+
In addition to offering the feature as an gRPC service, this project also includes the provision for distributing the same feature into a Ruby Gem.
|
16
16
|
This provision was added to fulfil [certain limitations](https://gitlab.com/gitlab-org/gitlab/-/issues/462359#note_1915874628). Here's the illustration representing
|
17
17
|
the approach:
|
18
18
|
|
@@ -34,13 +34,13 @@ the approach:
|
|
34
34
|
│ └── templates
|
35
35
|
│ ├── build.yml # CI jobs for building container image and ruby gems
|
36
36
|
│ ├── test.yml # CI jobs for running tests
|
37
|
-
│ └── release.yml # CI jobs for releases related to Ruby gem, GitLab releases
|
37
|
+
│ └── release.yml # CI jobs for releases related to Ruby gem, GitLab releases
|
38
38
|
├── lib
|
39
39
|
│ └── gitlab
|
40
|
-
│ └── secret_detection
|
40
|
+
│ └── secret_detection
|
41
41
|
│ ├── version.rb # Secret Detection Gem release version
|
42
42
|
│ ├── core/.. # Secret detection logic (most of it pulled from existing gem)
|
43
|
-
│ └── grpc
|
43
|
+
│ └── grpc
|
44
44
|
│ ├── generated/.. # gRPC generated files and secret detection gRPC service
|
45
45
|
│ └── scanner_service.rb # Secret detection gRPC service implementation
|
46
46
|
├── examples
|
@@ -76,7 +76,7 @@ Usage `make <command>`
|
|
76
76
|
|
77
77
|
## Generating a ruby gem
|
78
78
|
|
79
|
-
In the project directory, run `make gem` command in the terminal that builds a ruby gem(ex: `secret_detection-0.1.0.gem`) in the root of
|
79
|
+
In the project directory, run `make gem` command in the terminal that builds a ruby gem(ex: `secret_detection-0.1.0.gem`) in the root of
|
80
80
|
the project directory.
|
81
81
|
|
82
82
|
## Server
|
@@ -101,7 +101,7 @@ Pre-requisite: gRPC installed on your system (`brew install grpc`)
|
|
101
101
|
|
102
102
|
gRPC server port can be configured via `RPC_SERVER_PORT` environment variable
|
103
103
|
|
104
|
-
### Calling gRPC endpoints from terminal
|
104
|
+
### Calling gRPC endpoints from terminal
|
105
105
|
|
106
106
|
Pre-requisite:
|
107
107
|
- gRPC installed on your system (via `brew install grpc`)
|
@@ -188,7 +188,7 @@ You should see the following response as a result:
|
|
188
188
|
</details>
|
189
189
|
|
190
190
|
|
191
|
-
<details><summary>Example: Using
|
191
|
+
<details><summary>Example: Using Exclusions and Tags</summary>
|
192
192
|
|
193
193
|
```shell
|
194
194
|
$ grpcurl -d @ \
|
@@ -210,13 +210,13 @@ $ grpcurl -d @ \
|
|
210
210
|
"data": "GR1348941__DUMMY_RUNNER_TOKEN"
|
211
211
|
}
|
212
212
|
],
|
213
|
-
"
|
213
|
+
"exclusions": [
|
214
214
|
{
|
215
|
-
"
|
215
|
+
"exclusion_type": "EXCLUSION_TYPE_RULE",
|
216
216
|
"value": "gitlab_personal_access_token"
|
217
217
|
},
|
218
218
|
{
|
219
|
-
"
|
219
|
+
"exclusion_type": "EXCLUSION_TYPE_RAW_VALUE",
|
220
220
|
"value": "glrt-12345123451234512345"
|
221
221
|
}
|
222
222
|
],
|
@@ -252,17 +252,17 @@ You should see the following response as a result:
|
|
252
252
|
**RPC Method:** `gitlab.secret_detection.Scanner/ScanStream`
|
253
253
|
**Default Timeout(configurable):** per-request: `180 seconds`, per-payload: `30 seconds`
|
254
254
|
|
255
|
-
Bi-directional RPC streaming allows the server to receive a continuous stream of requests and respond with a
|
255
|
+
Bi-directional RPC streaming allows the server to receive a continuous stream of requests and respond with a
|
256
256
|
corresponding stream of responses as each request is processed. Unlike unary RPC calls, where requests are sent once and
|
257
|
-
the channel closes after a single response, bi-directional streaming keeps the RPC channel open to handle ongoing
|
258
|
-
requests and responses, enabling continuous communication between the client and server.
|
257
|
+
the channel closes after a single response, bi-directional streaming keeps the RPC channel open to handle ongoing
|
258
|
+
requests and responses, enabling continuous communication between the client and server.
|
259
259
|
|
260
260
|
|
261
261
|
<details><summary>Example</summary>
|
262
262
|
|
263
263
|
To try this out, we will accept the request input from STDIN. As on when you provide request input json in the terminal,
|
264
|
-
you should see a corresponding server response. The connection will continue to remain open for accepting
|
265
|
-
requests unless you explicitly close the connection with `Ctrl+C`.
|
264
|
+
you should see a corresponding server response. The connection will continue to remain open for accepting
|
265
|
+
requests unless you explicitly close the connection with `Ctrl+C`.
|
266
266
|
|
267
267
|
Here's the command to start an RPC streaming channel on the terminal:
|
268
268
|
|
@@ -300,7 +300,7 @@ you should immediately see the Server's response like below:
|
|
300
300
|
}
|
301
301
|
```
|
302
302
|
|
303
|
-
You may continue to provide more request inputs to the opened RPC channel to receive corresponding server responses. Features like
|
303
|
+
You may continue to provide more request inputs to the opened RPC channel to receive corresponding server responses. Features like Exclusions, Tag Filtering works same as outlined under Unary call section.
|
304
304
|
|
305
305
|
</details>
|
306
306
|
|
@@ -328,9 +328,9 @@ Secret Detection service's status can be tracked here: https://gitlab.com/gitlab
|
|
328
328
|
|
329
329
|
#### Changes made in the secret detection logic that were previously not present in the Gem
|
330
330
|
|
331
|
-
- [GitLab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
|
332
|
-
the `initialize()` method now accepts parsed rules instead of ruleset file path
|
331
|
+
- [GitLab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
|
332
|
+
the `initialize()` method now accepts parsed rules instead of ruleset file path
|
333
333
|
- [GitLab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
|
334
334
|
gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
|
335
|
-
- [GitLab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `
|
336
|
-
method to suport
|
335
|
+
- [GitLab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
|
336
|
+
method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
|
@@ -50,10 +50,10 @@ module GitLab
|
|
50
50
|
# +timeout+:: No of seconds(accepts floating point for smaller time values) to limit the total scan duration
|
51
51
|
# +payload_timeout+:: No of seconds(accepts floating point for smaller time values) to limit
|
52
52
|
# the scan duration on each payload
|
53
|
+
# +raw_value_exclusions:+:: Array of raw values to exclude from the scan.
|
53
54
|
# +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
|
54
55
|
# by its ID. For example: `gitlab_personal_access_token` for representing GitLab Personal Access
|
55
56
|
# Token. By default, no rule is excluded from the ruleset.
|
56
|
-
# +allow_values+:: Array of raw values to exclude from the scan.
|
57
57
|
# +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
|
58
58
|
# For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
|
59
59
|
# [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
|
@@ -68,8 +68,8 @@ module GitLab
|
|
68
68
|
payloads,
|
69
69
|
timeout: DEFAULT_SCAN_TIMEOUT_SECS,
|
70
70
|
payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
|
71
|
+
raw_value_exclusions: [],
|
71
72
|
rule_exclusions: [],
|
72
|
-
allow_values: [],
|
73
73
|
tags: DEFAULT_PATTERN_MATCHER_TAGS
|
74
74
|
)
|
75
75
|
|
@@ -90,7 +90,7 @@ module GitLab
|
|
90
90
|
secrets = run_scan(
|
91
91
|
payloads: matched_payloads, payload_timeout:,
|
92
92
|
pattern_matcher: build_pattern_matcher(tags:),
|
93
|
-
|
93
|
+
raw_value_exclusions:, rule_exclusions:
|
94
94
|
)
|
95
95
|
|
96
96
|
scan_status = overall_scan_status(secrets)
|
@@ -189,13 +189,13 @@ module GitLab
|
|
189
189
|
# literal values to exclude from the input before the scan, also SD rules to exclude during
|
190
190
|
# the scan when performed on the payloads.
|
191
191
|
def run_scan(
|
192
|
-
payloads:, payload_timeout:, pattern_matcher:,
|
192
|
+
payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [], rule_exclusions: [])
|
193
193
|
payloads.flat_map do |payload|
|
194
194
|
Timeout.timeout(payload_timeout) do
|
195
195
|
find_secrets_in_payload(
|
196
196
|
payload:,
|
197
197
|
pattern_matcher:,
|
198
|
-
|
198
|
+
raw_value_exclusions:, rule_exclusions:
|
199
199
|
)
|
200
200
|
end
|
201
201
|
rescue Timeout::Error => e
|
@@ -208,14 +208,14 @@ module GitLab
|
|
208
208
|
# Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
|
209
209
|
# literal values to exclude from the input before the scan, also SD rules to exclude during
|
210
210
|
# the scan.
|
211
|
-
def find_secrets_in_payload(payload:, pattern_matcher:,
|
211
|
+
def find_secrets_in_payload(payload:, pattern_matcher:, raw_value_exclusions: [], rule_exclusions: [])
|
212
212
|
findings = []
|
213
213
|
|
214
214
|
payload.data
|
215
215
|
.each_line($INPUT_RECORD_SEPARATOR, chomp: true)
|
216
216
|
.each_with_index do |line, index|
|
217
|
-
unless
|
218
|
-
|
217
|
+
unless raw_value_exclusions.empty?
|
218
|
+
raw_value_exclusions.each do |value|
|
219
219
|
line.gsub!(value, '') # replace input that doesn't contain allowed value in it
|
220
220
|
end
|
221
221
|
end
|
@@ -5,7 +5,7 @@
|
|
5
5
|
require 'google/protobuf'
|
6
6
|
|
7
7
|
|
8
|
-
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\
|
8
|
+
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe3\x04\n\x0cScanResponse\x12\x12\n\x05\x65rror\x18\x01 \x01(\tH\x00\x88\x01\x01\x12>\n\x07results\x18\x02 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12<\n\x06status\x18\x03 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x1a\xe9\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12<\n\x06status\x18\x02 \x01(\x0e\x32,.gitlab.secret_detection.ScanResponse.Status\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x12\n\x05\x65rror\x18\x06 \x01(\tH\x03\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_numberB\x08\n\x06_error\"\xca\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x42\x08\n\x06_error2\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitLab::SecretDetection::GRPCb\x06proto3"
|
9
9
|
|
10
10
|
pool = Google::Protobuf::DescriptorPool.generated_pool
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
@@ -15,8 +15,8 @@ module GitLab
|
|
15
15
|
module GRPC
|
16
16
|
ScanRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest").msgclass
|
17
17
|
ScanRequest::Payload = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.Payload").msgclass
|
18
|
-
ScanRequest::
|
19
|
-
ScanRequest::
|
18
|
+
ScanRequest::Exclusion = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.Exclusion").msgclass
|
19
|
+
ScanRequest::ExclusionType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest.ExclusionType").enummodule
|
20
20
|
ScanResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse").msgclass
|
21
21
|
ScanResponse::Finding = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse.Finding").msgclass
|
22
22
|
ScanResponse::Status = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanResponse.Status").enummodule
|
@@ -39,8 +39,8 @@ module GitLab
|
|
39
39
|
|
40
40
|
ERROR_MESSAGES = {
|
41
41
|
invalid_payload_fields: "Payload should not contain empty `id` and `data` fields",
|
42
|
-
|
43
|
-
|
42
|
+
exclusion_empty_value: "Exclusion value cannot be empty",
|
43
|
+
exclusion_invalid_type: "Invalid exclusion type",
|
44
44
|
invalid_timeout_range: "Timeout value should be > 0 and <= #{MAX_ALLOWED_TIMEOUT_SECONDS} seconds"
|
45
45
|
}.freeze
|
46
46
|
|
@@ -62,21 +62,22 @@ module GitLab
|
|
62
62
|
|
63
63
|
payloads = request.payloads.to_a
|
64
64
|
|
65
|
+
raw_value_exclusions = []
|
65
66
|
rule_exclusions = []
|
66
|
-
|
67
|
-
request.
|
68
|
-
case
|
69
|
-
when :
|
70
|
-
|
71
|
-
when :
|
72
|
-
|
67
|
+
|
68
|
+
request.exclusions&.each do |exclusion|
|
69
|
+
case exclusion.type
|
70
|
+
when :EXCLUSION_TYPE_RAW_VALUE
|
71
|
+
raw_value_exclusions << exclusion.value
|
72
|
+
when :EXCLUSION_TYPE_RULE
|
73
|
+
rule_exclusions << exclusion.value
|
73
74
|
end
|
74
75
|
end
|
75
76
|
|
76
77
|
result = scanner.secrets_scan(
|
77
78
|
payloads,
|
79
|
+
raw_value_exclusions:,
|
78
80
|
rule_exclusions:,
|
79
|
-
allow_values:,
|
80
81
|
tags: request.tags.to_a,
|
81
82
|
timeout: request.timeout_secs,
|
82
83
|
payload_timeout: request.payload_timeout_secs
|
@@ -103,10 +104,10 @@ module GitLab
|
|
103
104
|
# validates grpc request body
|
104
105
|
def validate_request(request)
|
105
106
|
# check for non-blank values and allowed types
|
106
|
-
request.
|
107
|
-
if
|
108
|
-
raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:
|
109
|
-
{ field: "
|
107
|
+
request.exclusions&.each do |exclusion|
|
108
|
+
if exclusion.value.empty?
|
109
|
+
raise ::GRPC::InvalidArgument.new(ERROR_MESSAGES[:exclusion_empty_value],
|
110
|
+
{ field: "exclusion.value" })
|
110
111
|
end
|
111
112
|
end
|
112
113
|
|
@@ -15,15 +15,15 @@ message ScanRequest {
|
|
15
15
|
}
|
16
16
|
|
17
17
|
// Either provide rule type or a particular value to allow during the scan
|
18
|
-
message
|
19
|
-
|
18
|
+
message Exclusion {
|
19
|
+
ExclusionType exclusion_type = 1;
|
20
20
|
string value = 2;
|
21
21
|
}
|
22
22
|
|
23
|
-
enum
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
enum ExclusionType {
|
24
|
+
EXCLUSION_TYPE_UNSPECIFIED = 0;
|
25
|
+
EXCLUSION_TYPE_RULE = 1; // Rule ID to exclude
|
26
|
+
EXCLUSION_TYPE_RAW_VALUE = 2; // Raw value to exclude
|
27
27
|
}
|
28
28
|
|
29
29
|
repeated Payload payloads = 1; // Array of payloads to scan
|
@@ -33,7 +33,7 @@ message ScanRequest {
|
|
33
33
|
// Scan timeout on each payload . Value is represented in seconds, accepts float values to represent smaller
|
34
34
|
// unit values. Default is 30 seconds.
|
35
35
|
optional float payload_timeout_secs = 3;
|
36
|
-
repeated
|
36
|
+
repeated Exclusion exclusions = 4; // Optional. Array of rule-types/raw-values to exclude from being considered during scan.
|
37
37
|
repeated string tags = 5; // Optional. Array of rule tags to consider for scan. Ex: ["gitlab_blocking"]
|
38
38
|
}
|
39
39
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-secret_detection
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- group::secret detection
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-09-
|
13
|
+
date: 2024-09-30 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: grpc
|