gitlab-secret_detection 0.5.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/gitlab/secret_detection/core/finding.rb +1 -1
- data/lib/gitlab/secret_detection/core/response.rb +3 -3
- data/lib/gitlab/secret_detection/core/ruleset.rb +1 -1
- data/lib/gitlab/secret_detection/core/scanner.rb +84 -6
- data/lib/gitlab/secret_detection/core/status.rb +1 -1
- data/lib/gitlab/secret_detection/core.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb +8 -8
- data/lib/gitlab/secret_detection/grpc/client/stream_request_enumerator.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb +2 -2
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb +4 -4
- data/lib/gitlab/secret_detection/grpc/scanner_service.rb +5 -5
- data/lib/gitlab/secret_detection/grpc.rb +1 -1
- data/lib/gitlab/secret_detection/utils/certificate.rb +3 -3
- data/lib/gitlab/secret_detection/utils/memoize.rb +2 -2
- data/lib/gitlab/secret_detection/utils.rb +1 -1
- data/lib/gitlab/secret_detection/version.rb +1 -1
- data/lib/gitlab/secret_detection.rb +1 -1
- data/lib/gitlab.rb +1 -1
- data/proto/secret_detection.proto +2 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc8541d1251962126cc8a11d661098fbe0713e610f849b554a50ab5cfe546a99
|
4
|
+
data.tar.gz: fc3eea2fbdbc233d0c8198a6cbf43c56d07d762777389f830422be26e90b714a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e7c0872064c6c85526ed8fb1d75a5d4ce11bbdf53498db161be7cd6fb54bf4f25e37b798ddd5e38c73762519f9b0d87d63e322706af7349181c832277c435421
|
7
|
+
data.tar.gz: 60b1d7ea4993b00cc2435bff4f0fae542d66c6ec2d1f58fafc47ce2850db0bb55c3a49b8b5e5fb9fa88fe277f941021792efc2892bd816109992438de6bfb888
|
data/README.md
CHANGED
@@ -329,9 +329,9 @@ Secret Detection service's status can be tracked here: https://gitlab.com/gitlab
|
|
329
329
|
|
330
330
|
#### Changes made in the secret detection logic that were previously not present in the Gem
|
331
331
|
|
332
|
-
- [
|
332
|
+
- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
|
333
333
|
the `initialize()` method now accepts parsed rules instead of ruleset file path
|
334
|
-
- [
|
334
|
+
- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
|
335
335
|
gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
|
336
|
-
- [
|
336
|
+
- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
|
337
337
|
method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Core
|
6
6
|
# Response is the data object returned by the scan operation with the following structure
|
7
7
|
#
|
8
|
-
# +status+:: One of values from
|
9
|
-
# +results+:: Array of
|
8
|
+
# +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
|
9
|
+
# +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
|
10
10
|
# +metadata+:: Hash object containing additional meta information about the response. It is currently used
|
11
11
|
# to embed more information on error.
|
12
12
|
class Response
|
@@ -4,8 +4,9 @@ require 're2'
|
|
4
4
|
require 'logger'
|
5
5
|
require 'timeout'
|
6
6
|
require 'English'
|
7
|
+
require 'parallel'
|
7
8
|
|
8
|
-
module
|
9
|
+
module Gitlab
|
9
10
|
module SecretDetection
|
10
11
|
module Core
|
11
12
|
# Scan is responsible for running Secret Detection scan operation
|
@@ -24,6 +25,14 @@ module GitLab
|
|
24
25
|
DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
|
25
26
|
# Tags used for creating default pattern matcher
|
26
27
|
DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
|
28
|
+
# Max no of child processes to spawn per request
|
29
|
+
# ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160
|
30
|
+
MAX_PROCS_PER_REQUEST = 5
|
31
|
+
# Minimum cumulative size of the payloads required to spawn and
|
32
|
+
# run the scan within a new subprocess.
|
33
|
+
MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB
|
34
|
+
# Whether to run scan in subprocesses or not. Default is false.
|
35
|
+
RUN_IN_SUBPROCESS = false
|
27
36
|
|
28
37
|
# Initializes the instance with logger along with following operations:
|
29
38
|
# 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
|
@@ -52,13 +61,20 @@ module GitLab
|
|
52
61
|
# the scan duration on each payload
|
53
62
|
# +raw_value_exclusions:+:: Array of raw values to exclude from the scan.
|
54
63
|
# +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
|
55
|
-
# by its ID. For example: `gitlab_personal_access_token` for representing
|
64
|
+
# by its ID. For example: `gitlab_personal_access_token` for representing Gitlab Personal Access
|
56
65
|
# Token. By default, no rule is excluded from the ruleset.
|
57
66
|
# +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
|
58
67
|
# For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
|
59
68
|
# [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
|
60
69
|
#
|
61
|
-
#
|
70
|
+
# NOTE:
|
71
|
+
# Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
|
72
|
+
# offloading regex operations on large payloads to sub-processes. However, it does not assure the improvement
|
73
|
+
# in the overall latency of the scan, specifically in the case of smaller payloads, where the overhead of
|
74
|
+
# forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based
|
75
|
+
# execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160.
|
76
|
+
#
|
77
|
+
# Returns an instance of Gitlab::SecretDetection::Core::Response by following below structure:
|
62
78
|
# {
|
63
79
|
# status: One of the Core::Status values
|
64
80
|
# results: [SecretDetection::Finding]
|
@@ -70,7 +86,8 @@ module GitLab
|
|
70
86
|
payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
|
71
87
|
raw_value_exclusions: [],
|
72
88
|
rule_exclusions: [],
|
73
|
-
tags: DEFAULT_PATTERN_MATCHER_TAGS
|
89
|
+
tags: DEFAULT_PATTERN_MATCHER_TAGS,
|
90
|
+
subprocess: RUN_IN_SUBPROCESS
|
74
91
|
)
|
75
92
|
|
76
93
|
return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
|
@@ -87,11 +104,13 @@ module GitLab
|
|
87
104
|
|
88
105
|
next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
|
89
106
|
|
90
|
-
|
107
|
+
scan_args = {
|
91
108
|
payloads: matched_payloads, payload_timeout:,
|
92
109
|
pattern_matcher: build_pattern_matcher(tags:),
|
93
110
|
raw_value_exclusions:, rule_exclusions:
|
94
|
-
|
111
|
+
}
|
112
|
+
|
113
|
+
secrets = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
|
95
114
|
|
96
115
|
scan_status = overall_scan_status(secrets)
|
97
116
|
|
@@ -205,6 +224,36 @@ module GitLab
|
|
205
224
|
end
|
206
225
|
end
|
207
226
|
|
227
|
+
def run_scan_within_subprocess(
|
228
|
+
payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [],
|
229
|
+
rule_exclusions: [])
|
230
|
+
payload_sizes = payloads.map(&:size)
|
231
|
+
grouped_payload_indices = group_by_chunk_size(payload_sizes)
|
232
|
+
|
233
|
+
grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
|
234
|
+
|
235
|
+
found_secrets = Parallel.flat_map(
|
236
|
+
grouped_payloads,
|
237
|
+
in_processes: MAX_PROCS_PER_REQUEST,
|
238
|
+
isolation: true # do not reuse sub-processes
|
239
|
+
) do |grouped_payload|
|
240
|
+
grouped_payload.flat_map do |payload|
|
241
|
+
Timeout.timeout(payload_timeout) do
|
242
|
+
find_secrets_in_payload(
|
243
|
+
payload:,
|
244
|
+
pattern_matcher:,
|
245
|
+
raw_value_exclusions:, rule_exclusions:
|
246
|
+
)
|
247
|
+
end
|
248
|
+
rescue Timeout::Error => e
|
249
|
+
logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
|
250
|
+
Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
found_secrets.freeze
|
255
|
+
end
|
256
|
+
|
208
257
|
# Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
|
209
258
|
# literal values to exclude from the input before the scan, also SD rules to exclude during
|
210
259
|
# the scan.
|
@@ -268,6 +317,35 @@ module GitLab
|
|
268
317
|
Core::Status::FOUND_WITH_ERRORS
|
269
318
|
end
|
270
319
|
end
|
320
|
+
|
321
|
+
# This method accepts an array of payload sizes(in bytes) and groups them into an array
|
322
|
+
# of arrays structure where each element is the group of indices of the input
|
323
|
+
# array whose cumulative payload sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+
|
324
|
+
def group_by_chunk_size(payload_size_arr)
|
325
|
+
cumulative_size = 0
|
326
|
+
chunk_indexes = []
|
327
|
+
chunk_idx_start = 0
|
328
|
+
|
329
|
+
payload_size_arr.each_with_index do |size, index|
|
330
|
+
cumulative_size += size
|
331
|
+
next unless cumulative_size >= MIN_CHUNK_SIZE_PER_PROC_BYTES
|
332
|
+
|
333
|
+
chunk_indexes << (chunk_idx_start..index).to_a
|
334
|
+
|
335
|
+
chunk_idx_start = index + 1
|
336
|
+
cumulative_size = 0
|
337
|
+
end
|
338
|
+
|
339
|
+
if cumulative_size.positive? && (chunk_idx_start < payload_size_arr.length)
|
340
|
+
chunk_indexes << if chunk_idx_start == payload_size_arr.length - 1
|
341
|
+
[chunk_idx_start]
|
342
|
+
else
|
343
|
+
(chunk_idx_start..payload_size_arr.length - 1).to_a
|
344
|
+
end
|
345
|
+
end
|
346
|
+
|
347
|
+
chunk_indexes
|
348
|
+
end
|
271
349
|
end
|
272
350
|
end
|
273
351
|
end
|
@@ -7,7 +7,7 @@ require_relative '../../core/status'
|
|
7
7
|
require_relative '../../utils'
|
8
8
|
require_relative './stream_request_enumerator'
|
9
9
|
|
10
|
-
module
|
10
|
+
module Gitlab
|
11
11
|
module SecretDetection
|
12
12
|
module GRPC
|
13
13
|
class Client
|
@@ -24,9 +24,9 @@ module GitLab
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
|
27
|
-
# this method transforms the gRPC response to +
|
27
|
+
# this method transforms the gRPC response to +Gitlab::SecretDetection::Core::Response+.
|
28
28
|
# Furthermore, any errors that are raised by the service will be translated to
|
29
|
-
# +
|
29
|
+
# +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
30
30
|
def run_scan(request:, auth_token:, extra_headers: {})
|
31
31
|
with_rescued_errors do
|
32
32
|
grpc_response = stub.scan(
|
@@ -42,13 +42,13 @@ module GitLab
|
|
42
42
|
# Triggers Secret Detection service's `/ScanStream` gRPC endpoint.
|
43
43
|
#
|
44
44
|
# To keep it consistent with SDS gem interface, this method transforms the gRPC response to
|
45
|
-
# +
|
46
|
-
# translated to +
|
45
|
+
# +Gitlab::SecretDetection::Core::Response+ type. Furthermore, any errors that are raised by the service will be
|
46
|
+
# translated to +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
47
47
|
#
|
48
48
|
# Note: If one of the stream requests result in an error, the stream will end immediately without processing the
|
49
49
|
# remaining requests.
|
50
50
|
def run_scan_stream(requests:, auth_token:, extra_headers: {})
|
51
|
-
request_stream =
|
51
|
+
request_stream = Gitlab::SecretDetection::GRPC::StreamRequestEnumerator.new(requests)
|
52
52
|
results = []
|
53
53
|
with_rescued_errors do
|
54
54
|
stub.scan_stream(
|
@@ -72,7 +72,7 @@ module GitLab
|
|
72
72
|
attr_reader :secure, :host, :compression
|
73
73
|
|
74
74
|
def stub
|
75
|
-
|
75
|
+
Gitlab::SecretDetection::GRPC::Scanner::Stub.new(
|
76
76
|
host,
|
77
77
|
channel_credentials,
|
78
78
|
channel_args:
|
@@ -100,7 +100,7 @@ module GitLab
|
|
100
100
|
def channel_credentials
|
101
101
|
return :this_channel_is_insecure unless secure
|
102
102
|
|
103
|
-
certs =
|
103
|
+
certs = Gitlab::SecretDetection::Utils::X509::Certificate.ca_certs_bundle
|
104
104
|
|
105
105
|
::GRPC::Core::ChannelCredentials.new(certs)
|
106
106
|
end
|
@@ -5,12 +5,12 @@
|
|
5
5
|
require 'google/protobuf'
|
6
6
|
|
7
7
|
|
8
|
-
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\
|
8
|
+
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
|
9
9
|
|
10
10
|
pool = Google::Protobuf::DescriptorPool.generated_pool
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
12
12
|
|
13
|
-
module
|
13
|
+
module Gitlab
|
14
14
|
module SecretDetection
|
15
15
|
module GRPC
|
16
16
|
ScanRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest").msgclass
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
2
|
-
# Source: secret_detection.proto for package '
|
2
|
+
# Source: secret_detection.proto for package 'Gitlab.SecretDetection.GRPC'
|
3
3
|
|
4
4
|
require 'grpc'
|
5
5
|
require 'secret_detection_pb'
|
6
6
|
|
7
|
-
module
|
7
|
+
module Gitlab
|
8
8
|
module SecretDetection
|
9
9
|
module GRPC
|
10
10
|
module Scanner
|
@@ -18,9 +18,9 @@ module GitLab
|
|
18
18
|
self.service_name = 'gitlab.secret_detection.Scanner'
|
19
19
|
|
20
20
|
# Runs secret detection scan for the given request
|
21
|
-
rpc :Scan, ::
|
21
|
+
rpc :Scan, ::Gitlab::SecretDetection::GRPC::ScanRequest, ::Gitlab::SecretDetection::GRPC::ScanResponse
|
22
22
|
# Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
|
23
|
-
rpc :ScanStream, stream(::
|
23
|
+
rpc :ScanStream, stream(::Gitlab::SecretDetection::GRPC::ScanRequest), stream(::Gitlab::SecretDetection::GRPC::ScanResponse)
|
24
24
|
end
|
25
25
|
|
26
26
|
Stub = Service.rpc_stub_class
|
@@ -27,7 +27,7 @@ class StreamEnumerator
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
module
|
30
|
+
module Gitlab
|
31
31
|
module SecretDetection
|
32
32
|
module GRPC
|
33
33
|
class ScannerService < Scanner::Service
|
@@ -89,21 +89,21 @@ module GitLab
|
|
89
89
|
end
|
90
90
|
|
91
91
|
findings = result.results&.map do |finding|
|
92
|
-
|
92
|
+
Gitlab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)
|
93
93
|
end
|
94
94
|
|
95
|
-
|
95
|
+
Gitlab::SecretDetection::GRPC::ScanResponse.new(
|
96
96
|
results: findings,
|
97
97
|
status: result.status
|
98
98
|
)
|
99
99
|
end
|
100
100
|
|
101
101
|
def scanner
|
102
|
-
@scanner ||=
|
102
|
+
@scanner ||= Gitlab::SecretDetection::Core::Scanner.new(rules:, logger:)
|
103
103
|
end
|
104
104
|
|
105
105
|
def rules
|
106
|
-
|
106
|
+
Gitlab::SecretDetection::Core::Ruleset.new.rules
|
107
107
|
end
|
108
108
|
|
109
109
|
# validates grpc request body
|
@@ -3,11 +3,11 @@
|
|
3
3
|
require 'openssl'
|
4
4
|
require_relative 'memoize'
|
5
5
|
|
6
|
-
module
|
6
|
+
module Gitlab
|
7
7
|
module SecretDetection
|
8
8
|
module Utils
|
9
9
|
module X509
|
10
|
-
# Pulled from
|
10
|
+
# Pulled from Gitlab.com source
|
11
11
|
# Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/lib/gitlab/x509/certificate.rb
|
12
12
|
class Certificate
|
13
13
|
CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
|
@@ -99,7 +99,7 @@ module GitLab
|
|
99
99
|
end
|
100
100
|
|
101
101
|
class << self
|
102
|
-
include ::
|
102
|
+
include ::Gitlab::SecretDetection::Utils::StrongMemoize
|
103
103
|
end
|
104
104
|
end
|
105
105
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Utils
|
6
6
|
# Pulled from GitLab.com source
|
@@ -16,7 +16,7 @@ module GitLab
|
|
16
16
|
#
|
17
17
|
# We could write it like:
|
18
18
|
#
|
19
|
-
# include
|
19
|
+
# include Gitlab::SecretDetection::Utils::StrongMemoize
|
20
20
|
#
|
21
21
|
# def trigger_from_token
|
22
22
|
# Ci::Trigger.find_by_token(params[:token].to_s)
|
data/lib/gitlab.rb
CHANGED
@@ -2,10 +2,10 @@ syntax = "proto3";
|
|
2
2
|
|
3
3
|
package gitlab.secret_detection;
|
4
4
|
|
5
|
-
/* We keep generated files within grpc namespace i.e
|
5
|
+
/* We keep generated files within grpc namespace i.e Gitlab::SecretDetection::GRPC
|
6
6
|
* so that these files are exported too in the Ruby Gem along with Core and GRPC logic.
|
7
7
|
*/
|
8
|
-
option ruby_package = "
|
8
|
+
option ruby_package = "Gitlab::SecretDetection::GRPC";
|
9
9
|
|
10
10
|
/* Request arg for triggering Scan/ScanStream method */
|
11
11
|
message ScanRequest {
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-secret_detection
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- group::secret detection
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-10-
|
13
|
+
date: 2024-10-08 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: grpc
|
@@ -40,6 +40,20 @@ dependencies:
|
|
40
40
|
- - '='
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: 1.63.0
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: parallel
|
45
|
+
requirement: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '1.19'
|
50
|
+
type: :runtime
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - "~>"
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '1.19'
|
43
57
|
- !ruby/object:Gem::Dependency
|
44
58
|
name: re2
|
45
59
|
requirement: !ruby/object:Gem::Requirement
|