gitlab-secret_detection 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/gitlab/secret_detection/core/finding.rb +1 -1
- data/lib/gitlab/secret_detection/core/response.rb +3 -3
- data/lib/gitlab/secret_detection/core/ruleset.rb +1 -1
- data/lib/gitlab/secret_detection/core/scanner.rb +84 -6
- data/lib/gitlab/secret_detection/core/status.rb +1 -1
- data/lib/gitlab/secret_detection/core.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb +8 -8
- data/lib/gitlab/secret_detection/grpc/client/stream_request_enumerator.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb +2 -2
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb +4 -4
- data/lib/gitlab/secret_detection/grpc/scanner_service.rb +5 -5
- data/lib/gitlab/secret_detection/grpc.rb +1 -1
- data/lib/gitlab/secret_detection/utils/certificate.rb +3 -3
- data/lib/gitlab/secret_detection/utils/memoize.rb +2 -2
- data/lib/gitlab/secret_detection/utils.rb +1 -1
- data/lib/gitlab/secret_detection/version.rb +1 -1
- data/lib/gitlab/secret_detection.rb +1 -1
- data/lib/gitlab.rb +1 -1
- data/proto/secret_detection.proto +2 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc8541d1251962126cc8a11d661098fbe0713e610f849b554a50ab5cfe546a99
|
4
|
+
data.tar.gz: fc3eea2fbdbc233d0c8198a6cbf43c56d07d762777389f830422be26e90b714a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e7c0872064c6c85526ed8fb1d75a5d4ce11bbdf53498db161be7cd6fb54bf4f25e37b798ddd5e38c73762519f9b0d87d63e322706af7349181c832277c435421
|
7
|
+
data.tar.gz: 60b1d7ea4993b00cc2435bff4f0fae542d66c6ec2d1f58fafc47ce2850db0bb55c3a49b8b5e5fb9fa88fe277f941021792efc2892bd816109992438de6bfb888
|
data/README.md
CHANGED
@@ -329,9 +329,9 @@ Secret Detection service's status can be tracked here: https://gitlab.com/gitlab
|
|
329
329
|
|
330
330
|
#### Changes made in the secret detection logic that were previously not present in the Gem
|
331
331
|
|
332
|
-
- [
|
332
|
+
- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
|
333
333
|
the `initialize()` method now accepts parsed rules instead of ruleset file path
|
334
|
-
- [
|
334
|
+
- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
|
335
335
|
gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
|
336
|
-
- [
|
336
|
+
- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
|
337
337
|
method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Core
|
6
6
|
# Response is the data object returned by the scan operation with the following structure
|
7
7
|
#
|
8
|
-
# +status+:: One of values from
|
9
|
-
# +results+:: Array of
|
8
|
+
# +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
|
9
|
+
# +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
|
10
10
|
# +metadata+:: Hash object containing additional meta information about the response. It is currently used
|
11
11
|
# to embed more information on error.
|
12
12
|
class Response
|
@@ -4,8 +4,9 @@ require 're2'
|
|
4
4
|
require 'logger'
|
5
5
|
require 'timeout'
|
6
6
|
require 'English'
|
7
|
+
require 'parallel'
|
7
8
|
|
8
|
-
module
|
9
|
+
module Gitlab
|
9
10
|
module SecretDetection
|
10
11
|
module Core
|
11
12
|
# Scan is responsible for running Secret Detection scan operation
|
@@ -24,6 +25,14 @@ module GitLab
|
|
24
25
|
DEFAULT_PAYLOAD_TIMEOUT_SECS = 30 # 30 seconds
|
25
26
|
# Tags used for creating default pattern matcher
|
26
27
|
DEFAULT_PATTERN_MATCHER_TAGS = ['gitlab_blocking'].freeze
|
28
|
+
# Max no of child processes to spawn per request
|
29
|
+
# ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160
|
30
|
+
MAX_PROCS_PER_REQUEST = 5
|
31
|
+
# Minimum cumulative size of the payloads required to spawn and
|
32
|
+
# run the scan within a new subprocess.
|
33
|
+
MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB
|
34
|
+
# Whether to run scan in subprocesses or not. Default is false.
|
35
|
+
RUN_IN_SUBPROCESS = false
|
27
36
|
|
28
37
|
# Initializes the instance with logger along with following operations:
|
29
38
|
# 1. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
|
@@ -52,13 +61,20 @@ module GitLab
|
|
52
61
|
# the scan duration on each payload
|
53
62
|
# +raw_value_exclusions:+:: Array of raw values to exclude from the scan.
|
54
63
|
# +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
|
55
|
-
# by its ID. For example: `gitlab_personal_access_token` for representing
|
64
|
+
# by its ID. For example: `gitlab_personal_access_token` for representing Gitlab Personal Access
|
56
65
|
# Token. By default, no rule is excluded from the ruleset.
|
57
66
|
# +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
|
58
67
|
# For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
|
59
68
|
# [`gitlab_blocking`] (+DEFAULT_PATTERN_MATCHER_TAGS+).
|
60
69
|
#
|
61
|
-
#
|
70
|
+
# NOTE:
|
71
|
+
# Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by
|
72
|
+
# offloading regex operations on large payloads to sub-processes. However, it does not assure the improvement
|
73
|
+
# in the overall latency of the scan, specifically in the case of smaller payloads, where the overhead of
|
74
|
+
# forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based
|
75
|
+
# execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160.
|
76
|
+
#
|
77
|
+
# Returns an instance of Gitlab::SecretDetection::Core::Response by following below structure:
|
62
78
|
# {
|
63
79
|
# status: One of the Core::Status values
|
64
80
|
# results: [SecretDetection::Finding]
|
@@ -70,7 +86,8 @@ module GitLab
|
|
70
86
|
payload_timeout: DEFAULT_PAYLOAD_TIMEOUT_SECS,
|
71
87
|
raw_value_exclusions: [],
|
72
88
|
rule_exclusions: [],
|
73
|
-
tags: DEFAULT_PATTERN_MATCHER_TAGS
|
89
|
+
tags: DEFAULT_PATTERN_MATCHER_TAGS,
|
90
|
+
subprocess: RUN_IN_SUBPROCESS
|
74
91
|
)
|
75
92
|
|
76
93
|
return Core::Response.new(Core::Status::INPUT_ERROR) unless validate_scan_input(payloads)
|
@@ -87,11 +104,13 @@ module GitLab
|
|
87
104
|
|
88
105
|
next Core::Response.new(Core::Status::NOT_FOUND) if matched_payloads.empty?
|
89
106
|
|
90
|
-
|
107
|
+
scan_args = {
|
91
108
|
payloads: matched_payloads, payload_timeout:,
|
92
109
|
pattern_matcher: build_pattern_matcher(tags:),
|
93
110
|
raw_value_exclusions:, rule_exclusions:
|
94
|
-
|
111
|
+
}
|
112
|
+
|
113
|
+
secrets = subprocess ? run_scan_within_subprocess(**scan_args) : run_scan(**scan_args)
|
95
114
|
|
96
115
|
scan_status = overall_scan_status(secrets)
|
97
116
|
|
@@ -205,6 +224,36 @@ module GitLab
|
|
205
224
|
end
|
206
225
|
end
|
207
226
|
|
227
|
+
def run_scan_within_subprocess(
|
228
|
+
payloads:, payload_timeout:, pattern_matcher:, raw_value_exclusions: [],
|
229
|
+
rule_exclusions: [])
|
230
|
+
payload_sizes = payloads.map(&:size)
|
231
|
+
grouped_payload_indices = group_by_chunk_size(payload_sizes)
|
232
|
+
|
233
|
+
grouped_payloads = grouped_payload_indices.map { |idx_arr| idx_arr.map { |i| payloads[i] } }
|
234
|
+
|
235
|
+
found_secrets = Parallel.flat_map(
|
236
|
+
grouped_payloads,
|
237
|
+
in_processes: MAX_PROCS_PER_REQUEST,
|
238
|
+
isolation: true # do not reuse sub-processes
|
239
|
+
) do |grouped_payload|
|
240
|
+
grouped_payload.flat_map do |payload|
|
241
|
+
Timeout.timeout(payload_timeout) do
|
242
|
+
find_secrets_in_payload(
|
243
|
+
payload:,
|
244
|
+
pattern_matcher:,
|
245
|
+
raw_value_exclusions:, rule_exclusions:
|
246
|
+
)
|
247
|
+
end
|
248
|
+
rescue Timeout::Error => e
|
249
|
+
logger.error "Secret Detection scan timed out on the payload(id:#{payload.id}): #{e}"
|
250
|
+
Core::Finding.new(payload.id, Core::Status::PAYLOAD_TIMEOUT)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
found_secrets.freeze
|
255
|
+
end
|
256
|
+
|
208
257
|
# Finds secrets in the given payload guarded with a timeout as a circuit breaker. It accepts
|
209
258
|
# literal values to exclude from the input before the scan, also SD rules to exclude during
|
210
259
|
# the scan.
|
@@ -268,6 +317,35 @@ module GitLab
|
|
268
317
|
Core::Status::FOUND_WITH_ERRORS
|
269
318
|
end
|
270
319
|
end
|
320
|
+
|
321
|
+
# This method accepts an array of payload sizes(in bytes) and groups them into an array
|
322
|
+
# of arrays structure where each element is the group of indices of the input
|
323
|
+
# array whose cumulative payload sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+
|
324
|
+
def group_by_chunk_size(payload_size_arr)
|
325
|
+
cumulative_size = 0
|
326
|
+
chunk_indexes = []
|
327
|
+
chunk_idx_start = 0
|
328
|
+
|
329
|
+
payload_size_arr.each_with_index do |size, index|
|
330
|
+
cumulative_size += size
|
331
|
+
next unless cumulative_size >= MIN_CHUNK_SIZE_PER_PROC_BYTES
|
332
|
+
|
333
|
+
chunk_indexes << (chunk_idx_start..index).to_a
|
334
|
+
|
335
|
+
chunk_idx_start = index + 1
|
336
|
+
cumulative_size = 0
|
337
|
+
end
|
338
|
+
|
339
|
+
if cumulative_size.positive? && (chunk_idx_start < payload_size_arr.length)
|
340
|
+
chunk_indexes << if chunk_idx_start == payload_size_arr.length - 1
|
341
|
+
[chunk_idx_start]
|
342
|
+
else
|
343
|
+
(chunk_idx_start..payload_size_arr.length - 1).to_a
|
344
|
+
end
|
345
|
+
end
|
346
|
+
|
347
|
+
chunk_indexes
|
348
|
+
end
|
271
349
|
end
|
272
350
|
end
|
273
351
|
end
|
@@ -7,7 +7,7 @@ require_relative '../../core/status'
|
|
7
7
|
require_relative '../../utils'
|
8
8
|
require_relative './stream_request_enumerator'
|
9
9
|
|
10
|
-
module
|
10
|
+
module Gitlab
|
11
11
|
module SecretDetection
|
12
12
|
module GRPC
|
13
13
|
class Client
|
@@ -24,9 +24,9 @@ module GitLab
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
|
27
|
-
# this method transforms the gRPC response to +
|
27
|
+
# this method transforms the gRPC response to +Gitlab::SecretDetection::Core::Response+.
|
28
28
|
# Furthermore, any errors that are raised by the service will be translated to
|
29
|
-
# +
|
29
|
+
# +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
30
30
|
def run_scan(request:, auth_token:, extra_headers: {})
|
31
31
|
with_rescued_errors do
|
32
32
|
grpc_response = stub.scan(
|
@@ -42,13 +42,13 @@ module GitLab
|
|
42
42
|
# Triggers Secret Detection service's `/ScanStream` gRPC endpoint.
|
43
43
|
#
|
44
44
|
# To keep it consistent with SDS gem interface, this method transforms the gRPC response to
|
45
|
-
# +
|
46
|
-
# translated to +
|
45
|
+
# +Gitlab::SecretDetection::Core::Response+ type. Furthermore, any errors that are raised by the service will be
|
46
|
+
# translated to +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
47
47
|
#
|
48
48
|
# Note: If one of the stream requests result in an error, the stream will end immediately without processing the
|
49
49
|
# remaining requests.
|
50
50
|
def run_scan_stream(requests:, auth_token:, extra_headers: {})
|
51
|
-
request_stream =
|
51
|
+
request_stream = Gitlab::SecretDetection::GRPC::StreamRequestEnumerator.new(requests)
|
52
52
|
results = []
|
53
53
|
with_rescued_errors do
|
54
54
|
stub.scan_stream(
|
@@ -72,7 +72,7 @@ module GitLab
|
|
72
72
|
attr_reader :secure, :host, :compression
|
73
73
|
|
74
74
|
def stub
|
75
|
-
|
75
|
+
Gitlab::SecretDetection::GRPC::Scanner::Stub.new(
|
76
76
|
host,
|
77
77
|
channel_credentials,
|
78
78
|
channel_args:
|
@@ -100,7 +100,7 @@ module GitLab
|
|
100
100
|
def channel_credentials
|
101
101
|
return :this_channel_is_insecure unless secure
|
102
102
|
|
103
|
-
certs =
|
103
|
+
certs = Gitlab::SecretDetection::Utils::X509::Certificate.ca_certs_bundle
|
104
104
|
|
105
105
|
::GRPC::Core::ChannelCredentials.new(certs)
|
106
106
|
end
|
@@ -5,12 +5,12 @@
|
|
5
5
|
require 'google/protobuf'
|
6
6
|
|
7
7
|
|
8
|
-
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\
|
8
|
+
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
|
9
9
|
|
10
10
|
pool = Google::Protobuf::DescriptorPool.generated_pool
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
12
12
|
|
13
|
-
module
|
13
|
+
module Gitlab
|
14
14
|
module SecretDetection
|
15
15
|
module GRPC
|
16
16
|
ScanRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest").msgclass
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
2
|
-
# Source: secret_detection.proto for package '
|
2
|
+
# Source: secret_detection.proto for package 'Gitlab.SecretDetection.GRPC'
|
3
3
|
|
4
4
|
require 'grpc'
|
5
5
|
require 'secret_detection_pb'
|
6
6
|
|
7
|
-
module
|
7
|
+
module Gitlab
|
8
8
|
module SecretDetection
|
9
9
|
module GRPC
|
10
10
|
module Scanner
|
@@ -18,9 +18,9 @@ module GitLab
|
|
18
18
|
self.service_name = 'gitlab.secret_detection.Scanner'
|
19
19
|
|
20
20
|
# Runs secret detection scan for the given request
|
21
|
-
rpc :Scan, ::
|
21
|
+
rpc :Scan, ::Gitlab::SecretDetection::GRPC::ScanRequest, ::Gitlab::SecretDetection::GRPC::ScanResponse
|
22
22
|
# Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
|
23
|
-
rpc :ScanStream, stream(::
|
23
|
+
rpc :ScanStream, stream(::Gitlab::SecretDetection::GRPC::ScanRequest), stream(::Gitlab::SecretDetection::GRPC::ScanResponse)
|
24
24
|
end
|
25
25
|
|
26
26
|
Stub = Service.rpc_stub_class
|
@@ -27,7 +27,7 @@ class StreamEnumerator
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
module
|
30
|
+
module Gitlab
|
31
31
|
module SecretDetection
|
32
32
|
module GRPC
|
33
33
|
class ScannerService < Scanner::Service
|
@@ -89,21 +89,21 @@ module GitLab
|
|
89
89
|
end
|
90
90
|
|
91
91
|
findings = result.results&.map do |finding|
|
92
|
-
|
92
|
+
Gitlab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)
|
93
93
|
end
|
94
94
|
|
95
|
-
|
95
|
+
Gitlab::SecretDetection::GRPC::ScanResponse.new(
|
96
96
|
results: findings,
|
97
97
|
status: result.status
|
98
98
|
)
|
99
99
|
end
|
100
100
|
|
101
101
|
def scanner
|
102
|
-
@scanner ||=
|
102
|
+
@scanner ||= Gitlab::SecretDetection::Core::Scanner.new(rules:, logger:)
|
103
103
|
end
|
104
104
|
|
105
105
|
def rules
|
106
|
-
|
106
|
+
Gitlab::SecretDetection::Core::Ruleset.new.rules
|
107
107
|
end
|
108
108
|
|
109
109
|
# validates grpc request body
|
@@ -3,11 +3,11 @@
|
|
3
3
|
require 'openssl'
|
4
4
|
require_relative 'memoize'
|
5
5
|
|
6
|
-
module
|
6
|
+
module Gitlab
|
7
7
|
module SecretDetection
|
8
8
|
module Utils
|
9
9
|
module X509
|
10
|
-
# Pulled from
|
10
|
+
# Pulled from Gitlab.com source
|
11
11
|
# Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/lib/gitlab/x509/certificate.rb
|
12
12
|
class Certificate
|
13
13
|
CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
|
@@ -99,7 +99,7 @@ module GitLab
|
|
99
99
|
end
|
100
100
|
|
101
101
|
class << self
|
102
|
-
include ::
|
102
|
+
include ::Gitlab::SecretDetection::Utils::StrongMemoize
|
103
103
|
end
|
104
104
|
end
|
105
105
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Utils
|
6
6
|
# Pulled from GitLab.com source
|
@@ -16,7 +16,7 @@ module GitLab
|
|
16
16
|
#
|
17
17
|
# We could write it like:
|
18
18
|
#
|
19
|
-
# include
|
19
|
+
# include Gitlab::SecretDetection::Utils::StrongMemoize
|
20
20
|
#
|
21
21
|
# def trigger_from_token
|
22
22
|
# Ci::Trigger.find_by_token(params[:token].to_s)
|
data/lib/gitlab.rb
CHANGED
@@ -2,10 +2,10 @@ syntax = "proto3";
|
|
2
2
|
|
3
3
|
package gitlab.secret_detection;
|
4
4
|
|
5
|
-
/* We keep generated files within grpc namespace i.e
|
5
|
+
/* We keep generated files within grpc namespace i.e Gitlab::SecretDetection::GRPC
|
6
6
|
* so that these files are exported too in the Ruby Gem along with Core and GRPC logic.
|
7
7
|
*/
|
8
|
-
option ruby_package = "
|
8
|
+
option ruby_package = "Gitlab::SecretDetection::GRPC";
|
9
9
|
|
10
10
|
/* Request arg for triggering Scan/ScanStream method */
|
11
11
|
message ScanRequest {
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-secret_detection
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- group::secret detection
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-10-
|
13
|
+
date: 2024-10-08 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: grpc
|
@@ -40,6 +40,20 @@ dependencies:
|
|
40
40
|
- - '='
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: 1.63.0
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: parallel
|
45
|
+
requirement: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '1.19'
|
50
|
+
type: :runtime
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - "~>"
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '1.19'
|
43
57
|
- !ruby/object:Gem::Dependency
|
44
58
|
name: re2
|
45
59
|
requirement: !ruby/object:Gem::Requirement
|