gitlab-secret_detection 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/gitlab/secret_detection/core/finding.rb +1 -1
- data/lib/gitlab/secret_detection/core/response.rb +3 -3
- data/lib/gitlab/secret_detection/core/ruleset.rb +1 -1
- data/lib/gitlab/secret_detection/core/scanner.rb +3 -3
- data/lib/gitlab/secret_detection/core/status.rb +1 -1
- data/lib/gitlab/secret_detection/core.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/client/grpc_client.rb +8 -8
- data/lib/gitlab/secret_detection/grpc/client/stream_request_enumerator.rb +1 -1
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_pb.rb +2 -2
- data/lib/gitlab/secret_detection/grpc/generated/secret_detection_services_pb.rb +4 -4
- data/lib/gitlab/secret_detection/grpc/scanner_service.rb +5 -5
- data/lib/gitlab/secret_detection/grpc.rb +1 -1
- data/lib/gitlab/secret_detection/utils/certificate.rb +3 -3
- data/lib/gitlab/secret_detection/utils/memoize.rb +2 -2
- data/lib/gitlab/secret_detection/utils.rb +1 -1
- data/lib/gitlab/secret_detection/version.rb +1 -1
- data/lib/gitlab/secret_detection.rb +1 -1
- data/lib/gitlab.rb +1 -1
- data/proto/secret_detection.proto +2 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff032818993f54b85b958ad43293df4e9112ad20b9d68ab654abf74ebeb778a7
|
4
|
+
data.tar.gz: e0556ec3bc97e6973bbb6c8ba4e041897eccfbb37e23d4af16ff4fccd4850eb0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ae963ac226ddab5d154879c078b44581f859232175ecf69269818a0e880368855469f10265a63d408089b5da7d56e37f51a865bbd9efc66ddd5474d1b718af0a
|
7
|
+
data.tar.gz: ff4c2bff936d88972a1e51563e2f6a0fb61f3c431213058fccae5e7abd18ac72636e37099a0dece9f083424795078bf2b69dd224aa88b27e6752bed1ae9d37ee
|
data/README.md
CHANGED
@@ -329,9 +329,9 @@ Secret Detection service's status can be tracked here: https://gitlab.com/gitlab
|
|
329
329
|
|
330
330
|
#### Changes made in the secret detection logic that were previously not present in the Gem
|
331
331
|
|
332
|
-
- [
|
332
|
+
- [Gitlab::SecretDetection::Core::Scanner#initialize(...)](lib/gitlab/secret_detection/core/scanner.rb): To reuse the logic of ruleset parsing from a file source, we parse the ruleset file at once and pass the parsed rules around. So,
|
333
333
|
the `initialize()` method now accepts parsed rules instead of ruleset file path
|
334
|
-
- [
|
334
|
+
- [Gitlab::SecretDetection::Core::Status](lib/gitlab/secret_detection/core/status.rb): `NOT_FOUND` status moved from `0` to `7` since
|
335
335
|
gRPC reserves `0` for enums. We need to reflect this change on the Rails side too
|
336
|
-
- [
|
336
|
+
- [Gitlab::SecretDetection::Core::Scanner#scan(...)](lib/gitlab/secret_detection/core/scanner.rb): Introduced `rule_exclusions`, `raw_value_exclusions` and `tags` args to `scan(..)`
|
337
337
|
method to suport [exclusions](https://gitlab.com/groups/gitlab-org/-/epics/14315) feature.
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Core
|
6
6
|
# Response is the data object returned by the scan operation with the following structure
|
7
7
|
#
|
8
|
-
# +status+:: One of values from
|
9
|
-
# +results+:: Array of
|
8
|
+
# +status+:: One of values from Gitlab::SecretDetection::Core::Status indicating the scan operation's status
|
9
|
+
# +results+:: Array of Gitlab::SecretDetection::Core::Finding values. Default value is nil.
|
10
10
|
# +metadata+:: Hash object containing additional meta information about the response. It is currently used
|
11
11
|
# to embed more information on error.
|
12
12
|
class Response
|
@@ -6,7 +6,7 @@ require 'timeout'
|
|
6
6
|
require 'English'
|
7
7
|
require 'parallel'
|
8
8
|
|
9
|
-
module
|
9
|
+
module Gitlab
|
10
10
|
module SecretDetection
|
11
11
|
module Core
|
12
12
|
# Scan is responsible for running Secret Detection scan operation
|
@@ -61,7 +61,7 @@ module GitLab
|
|
61
61
|
# the scan duration on each payload
|
62
62
|
# +raw_value_exclusions:+:: Array of raw values to exclude from the scan.
|
63
63
|
# +rule_exclusions+:: Array of rules to exclude from the ruleset used for the scan. Each rule is represented
|
64
|
-
# by its ID. For example: `gitlab_personal_access_token` for representing
|
64
|
+
# by its ID. For example: `gitlab_personal_access_token` for representing Gitlab Personal Access
|
65
65
|
# Token. By default, no rule is excluded from the ruleset.
|
66
66
|
# +tags+:: Array of tag values to filter from the default ruleset when determining the rules used for the scan.
|
67
67
|
# For example: Add `gitlab_blocking` to include only rules for Push Protection. Defaults to
|
@@ -74,7 +74,7 @@ module GitLab
|
|
74
74
|
# forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based
|
75
75
|
# execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160.
|
76
76
|
#
|
77
|
-
# Returns an instance of
|
77
|
+
# Returns an instance of Gitlab::SecretDetection::Core::Response by following below structure:
|
78
78
|
# {
|
79
79
|
# status: One of the Core::Status values
|
80
80
|
# results: [SecretDetection::Finding]
|
@@ -7,7 +7,7 @@ require_relative '../../core/status'
|
|
7
7
|
require_relative '../../utils'
|
8
8
|
require_relative './stream_request_enumerator'
|
9
9
|
|
10
|
-
module
|
10
|
+
module Gitlab
|
11
11
|
module SecretDetection
|
12
12
|
module GRPC
|
13
13
|
class Client
|
@@ -24,9 +24,9 @@ module GitLab
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Triggers Secret Detection service's `/Scan` gRPC endpoint. To keep it consistent with SDS gem interface,
|
27
|
-
# this method transforms the gRPC response to +
|
27
|
+
# this method transforms the gRPC response to +Gitlab::SecretDetection::Core::Response+.
|
28
28
|
# Furthermore, any errors that are raised by the service will be translated to
|
29
|
-
# +
|
29
|
+
# +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
30
30
|
def run_scan(request:, auth_token:, extra_headers: {})
|
31
31
|
with_rescued_errors do
|
32
32
|
grpc_response = stub.scan(
|
@@ -42,13 +42,13 @@ module GitLab
|
|
42
42
|
# Triggers Secret Detection service's `/ScanStream` gRPC endpoint.
|
43
43
|
#
|
44
44
|
# To keep it consistent with SDS gem interface, this method transforms the gRPC response to
|
45
|
-
# +
|
46
|
-
# translated to +
|
45
|
+
# +Gitlab::SecretDetection::Core::Response+ type. Furthermore, any errors that are raised by the service will be
|
46
|
+
# translated to +Gitlab::SecretDetection::Core::Response+ type by assiging a appropriate +status+ value to it.
|
47
47
|
#
|
48
48
|
# Note: If one of the stream requests result in an error, the stream will end immediately without processing the
|
49
49
|
# remaining requests.
|
50
50
|
def run_scan_stream(requests:, auth_token:, extra_headers: {})
|
51
|
-
request_stream =
|
51
|
+
request_stream = Gitlab::SecretDetection::GRPC::StreamRequestEnumerator.new(requests)
|
52
52
|
results = []
|
53
53
|
with_rescued_errors do
|
54
54
|
stub.scan_stream(
|
@@ -72,7 +72,7 @@ module GitLab
|
|
72
72
|
attr_reader :secure, :host, :compression
|
73
73
|
|
74
74
|
def stub
|
75
|
-
|
75
|
+
Gitlab::SecretDetection::GRPC::Scanner::Stub.new(
|
76
76
|
host,
|
77
77
|
channel_credentials,
|
78
78
|
channel_args:
|
@@ -100,7 +100,7 @@ module GitLab
|
|
100
100
|
def channel_credentials
|
101
101
|
return :this_channel_is_insecure unless secure
|
102
102
|
|
103
|
-
certs =
|
103
|
+
certs = Gitlab::SecretDetection::Utils::X509::Certificate.ca_certs_bundle
|
104
104
|
|
105
105
|
::GRPC::Core::ChannelCredentials.new(certs)
|
106
106
|
end
|
@@ -5,12 +5,12 @@
|
|
5
5
|
require 'google/protobuf'
|
6
6
|
|
7
7
|
|
8
|
-
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\
|
8
|
+
descriptor_data = "\n\x16secret_detection.proto\x12\x17gitlab.secret_detection\"\xfc\x03\n\x0bScanRequest\x12>\n\x08payloads\x18\x01 \x03(\x0b\x32,.gitlab.secret_detection.ScanRequest.Payload\x12\x19\n\x0ctimeout_secs\x18\x02 \x01(\x02H\x00\x88\x01\x01\x12!\n\x14payload_timeout_secs\x18\x03 \x01(\x02H\x01\x88\x01\x01\x12\x42\n\nexclusions\x18\x04 \x03(\x0b\x32..gitlab.secret_detection.ScanRequest.Exclusion\x12\x0c\n\x04tags\x18\x05 \x03(\t\x1a#\n\x07Payload\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x1a\x66\n\tExclusion\x12J\n\x0e\x65xclusion_type\x18\x01 \x01(\x0e\x32\x32.gitlab.secret_detection.ScanRequest.ExclusionType\x12\r\n\x05value\x18\x02 \x01(\t\"f\n\rExclusionType\x12\x1e\n\x1a\x45XCLUSION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XCLUSION_TYPE_RULE\x10\x01\x12\x1c\n\x18\x45XCLUSION_TYPE_RAW_VALUE\x10\x02\x42\x0f\n\r_timeout_secsB\x17\n\x15_payload_timeout_secs\"\xe2\x03\n\x0cScanResponse\x12>\n\x07results\x18\x01 \x03(\x0b\x32-.gitlab.secret_detection.ScanResponse.Finding\x12\x0e\n\x06status\x18\x02 \x01(\x05\x1a\x9d\x01\n\x07\x46inding\x12\x12\n\npayload_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\x05\x12\x11\n\x04type\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0b\x64\x65scription\x18\x04 \x01(\tH\x01\x88\x01\x01\x12\x18\n\x0bline_number\x18\x05 \x01(\x05H\x02\x88\x01\x01\x42\x07\n\x05_typeB\x0e\n\x0c_descriptionB\x0e\n\x0c_line_number\"\xe1\x01\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x10\n\x0cSTATUS_FOUND\x10\x01\x12\x1c\n\x18STATUS_FOUND_WITH_ERRORS\x10\x02\x12\x17\n\x13STATUS_SCAN_TIMEOUT\x10\x03\x12\x1a\n\x16STATUS_PAYLOAD_TIMEOUT\x10\x04\x12\x15\n\x11STATUS_SCAN_ERROR\x10\x05\x12\x16\n\x12STATUS_INPUT_ERROR\x10\x06\x12\x14\n\x10STATUS_NOT_FOUND\x10\x07\x12\x15\n\x11STATUS_AUTH_ERROR\x10\x08\x32\xc1\x01\n\x07Scanner\x12U\n\x04Scan\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00\x12_\n\nScanStream\x12$.gitlab.secret_detection.ScanRequest\x1a%.gitlab.secret_detection.ScanResponse\"\x00(\x01\x30\x01\x42 \xea\x02\x1dGitlab::SecretDetection::GRPCb\x06proto3"
|
9
9
|
|
10
10
|
pool = Google::Protobuf::DescriptorPool.generated_pool
|
11
11
|
pool.add_serialized_file(descriptor_data)
|
12
12
|
|
13
|
-
module
|
13
|
+
module Gitlab
|
14
14
|
module SecretDetection
|
15
15
|
module GRPC
|
16
16
|
ScanRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitlab.secret_detection.ScanRequest").msgclass
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
2
|
-
# Source: secret_detection.proto for package '
|
2
|
+
# Source: secret_detection.proto for package 'Gitlab.SecretDetection.GRPC'
|
3
3
|
|
4
4
|
require 'grpc'
|
5
5
|
require 'secret_detection_pb'
|
6
6
|
|
7
|
-
module
|
7
|
+
module Gitlab
|
8
8
|
module SecretDetection
|
9
9
|
module GRPC
|
10
10
|
module Scanner
|
@@ -18,9 +18,9 @@ module GitLab
|
|
18
18
|
self.service_name = 'gitlab.secret_detection.Scanner'
|
19
19
|
|
20
20
|
# Runs secret detection scan for the given request
|
21
|
-
rpc :Scan, ::
|
21
|
+
rpc :Scan, ::Gitlab::SecretDetection::GRPC::ScanRequest, ::Gitlab::SecretDetection::GRPC::ScanResponse
|
22
22
|
# Runs bi-directional streaming of scans for the given stream of requests with a stream of responses
|
23
|
-
rpc :ScanStream, stream(::
|
23
|
+
rpc :ScanStream, stream(::Gitlab::SecretDetection::GRPC::ScanRequest), stream(::Gitlab::SecretDetection::GRPC::ScanResponse)
|
24
24
|
end
|
25
25
|
|
26
26
|
Stub = Service.rpc_stub_class
|
@@ -27,7 +27,7 @@ class StreamEnumerator
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
module
|
30
|
+
module Gitlab
|
31
31
|
module SecretDetection
|
32
32
|
module GRPC
|
33
33
|
class ScannerService < Scanner::Service
|
@@ -89,21 +89,21 @@ module GitLab
|
|
89
89
|
end
|
90
90
|
|
91
91
|
findings = result.results&.map do |finding|
|
92
|
-
|
92
|
+
Gitlab::SecretDetection::GRPC::ScanResponse::Finding.new(**finding.to_h)
|
93
93
|
end
|
94
94
|
|
95
|
-
|
95
|
+
Gitlab::SecretDetection::GRPC::ScanResponse.new(
|
96
96
|
results: findings,
|
97
97
|
status: result.status
|
98
98
|
)
|
99
99
|
end
|
100
100
|
|
101
101
|
def scanner
|
102
|
-
@scanner ||=
|
102
|
+
@scanner ||= Gitlab::SecretDetection::Core::Scanner.new(rules:, logger:)
|
103
103
|
end
|
104
104
|
|
105
105
|
def rules
|
106
|
-
|
106
|
+
Gitlab::SecretDetection::Core::Ruleset.new.rules
|
107
107
|
end
|
108
108
|
|
109
109
|
# validates grpc request body
|
@@ -3,11 +3,11 @@
|
|
3
3
|
require 'openssl'
|
4
4
|
require_relative 'memoize'
|
5
5
|
|
6
|
-
module
|
6
|
+
module Gitlab
|
7
7
|
module SecretDetection
|
8
8
|
module Utils
|
9
9
|
module X509
|
10
|
-
# Pulled from
|
10
|
+
# Pulled from Gitlab.com source
|
11
11
|
# Link: https://gitlab.com/gitlab-org/gitlab/-/blob/4713a798f997389f04e442db3d1d8349a39d5d46/lib/gitlab/x509/certificate.rb
|
12
12
|
class Certificate
|
13
13
|
CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/
|
@@ -99,7 +99,7 @@ module GitLab
|
|
99
99
|
end
|
100
100
|
|
101
101
|
class << self
|
102
|
-
include ::
|
102
|
+
include ::Gitlab::SecretDetection::Utils::StrongMemoize
|
103
103
|
end
|
104
104
|
end
|
105
105
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Gitlab
|
4
4
|
module SecretDetection
|
5
5
|
module Utils
|
6
6
|
# Pulled from GitLab.com source
|
@@ -16,7 +16,7 @@ module GitLab
|
|
16
16
|
#
|
17
17
|
# We could write it like:
|
18
18
|
#
|
19
|
-
# include
|
19
|
+
# include Gitlab::SecretDetection::Utils::StrongMemoize
|
20
20
|
#
|
21
21
|
# def trigger_from_token
|
22
22
|
# Ci::Trigger.find_by_token(params[:token].to_s)
|
data/lib/gitlab.rb
CHANGED
@@ -2,10 +2,10 @@ syntax = "proto3";
|
|
2
2
|
|
3
3
|
package gitlab.secret_detection;
|
4
4
|
|
5
|
-
/* We keep generated files within grpc namespace i.e
|
5
|
+
/* We keep generated files within grpc namespace i.e Gitlab::SecretDetection::GRPC
|
6
6
|
* so that these files are exported too in the Ruby Gem along with Core and GRPC logic.
|
7
7
|
*/
|
8
|
-
option ruby_package = "
|
8
|
+
option ruby_package = "Gitlab::SecretDetection::GRPC";
|
9
9
|
|
10
10
|
/* Request arg for triggering Scan/ScanStream method */
|
11
11
|
message ScanRequest {
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-secret_detection
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- group::secret detection
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-10-
|
13
|
+
date: 2024-10-08 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: grpc
|