datadog-ci 1.0.0.beta1 → 1.0.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/README.md +37 -46
- data/lib/datadog/ci/configuration/components.rb +43 -9
- data/lib/datadog/ci/configuration/settings.rb +6 -0
- data/lib/datadog/ci/contrib/cucumber/formatter.rb +9 -7
- data/lib/datadog/ci/contrib/cucumber/patcher.rb +3 -0
- data/lib/datadog/ci/contrib/cucumber/step.rb +27 -0
- data/lib/datadog/ci/contrib/minitest/hooks.rb +4 -2
- data/lib/datadog/ci/contrib/rspec/example.rb +9 -5
- data/lib/datadog/ci/ext/environment/providers/local_git.rb +8 -79
- data/lib/datadog/ci/ext/environment.rb +11 -16
- data/lib/datadog/ci/ext/settings.rb +1 -0
- data/lib/datadog/ci/ext/test.rb +5 -0
- data/lib/datadog/ci/ext/transport.rb +8 -0
- data/lib/datadog/ci/git/local_repository.rb +238 -0
- data/lib/datadog/ci/git/packfiles.rb +70 -0
- data/lib/datadog/ci/git/search_commits.rb +77 -0
- data/lib/datadog/ci/git/tree_uploader.rb +90 -0
- data/lib/datadog/ci/git/upload_packfile.rb +66 -0
- data/lib/datadog/ci/git/user.rb +29 -0
- data/lib/datadog/ci/itr/coverage/event.rb +18 -1
- data/lib/datadog/ci/itr/coverage/writer.rb +108 -0
- data/lib/datadog/ci/itr/runner.rb +120 -11
- data/lib/datadog/ci/itr/skippable.rb +106 -0
- data/lib/datadog/ci/span.rb +9 -0
- data/lib/datadog/ci/test.rb +19 -12
- data/lib/datadog/ci/test_module.rb +2 -2
- data/lib/datadog/ci/test_session.rb +2 -2
- data/lib/datadog/ci/test_suite.rb +2 -2
- data/lib/datadog/ci/test_visibility/null_recorder.rb +4 -1
- data/lib/datadog/ci/test_visibility/recorder.rb +47 -9
- data/lib/datadog/ci/test_visibility/transport.rb +1 -1
- data/lib/datadog/ci/transport/http.rb +24 -4
- data/lib/datadog/ci/transport/remote_settings_api.rb +12 -6
- data/lib/datadog/ci/utils/configuration.rb +2 -2
- data/lib/datadog/ci/utils/git.rb +6 -67
- data/lib/datadog/ci/utils/parsing.rb +16 -0
- data/lib/datadog/ci/utils/test_run.rb +13 -0
- data/lib/datadog/ci/version.rb +1 -1
- data/lib/datadog/ci/worker.rb +35 -0
- data/lib/datadog/ci.rb +4 -0
- metadata +15 -4
@@ -3,6 +3,8 @@
|
|
3
3
|
require_relative "git"
|
4
4
|
require_relative "environment/extractor"
|
5
5
|
|
6
|
+
require_relative "../utils/git"
|
7
|
+
|
6
8
|
module Datadog
|
7
9
|
module CI
|
8
10
|
module Ext
|
@@ -21,8 +23,6 @@ module Datadog
|
|
21
23
|
TAG_NODE_NAME = "ci.node.name"
|
22
24
|
TAG_CI_ENV_VARS = "_dd.ci.env_vars"
|
23
25
|
|
24
|
-
HEX_NUMBER_REGEXP = /[0-9a-f]{40}/i.freeze
|
25
|
-
|
26
26
|
module_function
|
27
27
|
|
28
28
|
def tags(env)
|
@@ -57,24 +57,19 @@ module Datadog
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def validate_git_sha(git_sha)
|
60
|
-
|
60
|
+
return if Utils::Git.valid_commit_sha?(git_sha)
|
61
61
|
|
62
|
-
|
63
|
-
message += " No value was set and no SHA was automatically extracted."
|
64
|
-
Datadog.logger.error(message)
|
65
|
-
return
|
66
|
-
end
|
62
|
+
message = "DD_GIT_COMMIT_SHA must be a full-length git SHA."
|
67
63
|
|
68
|
-
if git_sha.
|
69
|
-
|
70
|
-
|
71
|
-
|
64
|
+
message += if git_sha.nil? || git_sha.empty?
|
65
|
+
" No value was set and no SHA was automatically extracted."
|
66
|
+
elsif git_sha.length < Git::SHA_LENGTH
|
67
|
+
" Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}."
|
68
|
+
else
|
69
|
+
" Expected SHA to be a valid HEX number, got #{git_sha}."
|
72
70
|
end
|
73
71
|
|
74
|
-
|
75
|
-
message += " Expected SHA to be a valid HEX number, got #{git_sha}."
|
76
|
-
Datadog.logger.error(message)
|
77
|
-
end
|
72
|
+
Datadog.logger.error(message)
|
78
73
|
end
|
79
74
|
end
|
80
75
|
end
|
@@ -11,6 +11,7 @@ module Datadog
|
|
11
11
|
ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
|
12
12
|
ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
|
13
13
|
ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED"
|
14
|
+
ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED"
|
14
15
|
|
15
16
|
# Source: https://docs.datadoghq.com/getting_started/site/
|
16
17
|
DD_SITE_ALLOWLIST = [
|
data/lib/datadog/ci/ext/test.rb
CHANGED
@@ -26,6 +26,9 @@ module Datadog
|
|
26
26
|
# ITR tags
|
27
27
|
TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled"
|
28
28
|
TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type"
|
29
|
+
TAG_ITR_TEST_SKIPPING_COUNT = "test.itr.tests_skipping.count"
|
30
|
+
TAG_ITR_SKIPPED_BY_ITR = "test.skipped_by_itr"
|
31
|
+
TAG_ITR_TESTS_SKIPPED = "_dd.ci.itr.tests_skipped"
|
29
32
|
|
30
33
|
# Code coverage tags
|
31
34
|
TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
|
@@ -43,6 +46,7 @@ module Datadog
|
|
43
46
|
# Environment runtime tags
|
44
47
|
TAG_OS_ARCHITECTURE = "os.architecture"
|
45
48
|
TAG_OS_PLATFORM = "os.platform"
|
49
|
+
TAG_OS_VERSION = "os.version"
|
46
50
|
TAG_RUNTIME_NAME = "runtime.name"
|
47
51
|
TAG_RUNTIME_VERSION = "runtime.version"
|
48
52
|
|
@@ -53,6 +57,7 @@ module Datadog
|
|
53
57
|
# could be either "test" or "suite" depending on whether we skip individual tests or whole suites
|
54
58
|
# we use test skipping for Ruby
|
55
59
|
ITR_TEST_SKIPPING_MODE = "test"
|
60
|
+
ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
|
56
61
|
|
57
62
|
# test status as recognized by Datadog
|
58
63
|
module Status
|
@@ -27,6 +27,7 @@ module Datadog
|
|
27
27
|
TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
|
28
28
|
|
29
29
|
DD_API_HOST_PREFIX = "api"
|
30
|
+
|
30
31
|
DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
|
31
32
|
DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings"
|
32
33
|
DD_API_SETTINGS_RESPONSE_DIG_KEYS = %w[data attributes].freeze
|
@@ -36,6 +37,13 @@ module Datadog
|
|
36
37
|
DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
|
37
38
|
DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
|
38
39
|
|
40
|
+
DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
|
41
|
+
|
42
|
+
DD_API_GIT_UPLOAD_PACKFILE_PATH = "/api/v2/git/repository/packfile"
|
43
|
+
|
44
|
+
DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable"
|
45
|
+
DD_API_SKIPPABLE_TESTS_TYPE = "test_params"
|
46
|
+
|
39
47
|
CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
|
40
48
|
CONTENT_TYPE_JSON = "application/json"
|
41
49
|
CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"
|
@@ -0,0 +1,238 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "open3"
|
4
|
+
require "pathname"
|
5
|
+
|
6
|
+
require_relative "user"
|
7
|
+
|
8
|
+
module Datadog
|
9
|
+
module CI
|
10
|
+
module Git
|
11
|
+
module LocalRepository
|
12
|
+
COMMAND_RETRY_COUNT = 3
|
13
|
+
|
14
|
+
def self.root
|
15
|
+
return @root if defined?(@root)
|
16
|
+
|
17
|
+
@root = git_root || Dir.pwd
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.relative_to_root(path)
|
21
|
+
return "" if path.nil?
|
22
|
+
|
23
|
+
root_path = root
|
24
|
+
return path if root_path.nil?
|
25
|
+
|
26
|
+
path = Pathname.new(File.expand_path(path))
|
27
|
+
root_path = Pathname.new(root_path)
|
28
|
+
|
29
|
+
path.relative_path_from(root_path).to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.repository_name
|
33
|
+
return @repository_name if defined?(@repository_name)
|
34
|
+
|
35
|
+
git_remote_url = git_repository_url
|
36
|
+
|
37
|
+
# return git repository name from remote url without .git extension
|
38
|
+
last_path_segment = git_remote_url.split("/").last if git_remote_url
|
39
|
+
@repository_name = last_path_segment.gsub(".git", "") if last_path_segment
|
40
|
+
@repository_name ||= current_folder_name
|
41
|
+
rescue => e
|
42
|
+
log_failure(e, "git repository name")
|
43
|
+
@repository_name = current_folder_name
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.current_folder_name
|
47
|
+
File.basename(root)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.git_repository_url
|
51
|
+
exec_git_command("git ls-remote --get-url")
|
52
|
+
rescue => e
|
53
|
+
log_failure(e, "git repository url")
|
54
|
+
nil
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.git_root
|
58
|
+
exec_git_command("git rev-parse --show-toplevel")
|
59
|
+
rescue => e
|
60
|
+
log_failure(e, "git root path")
|
61
|
+
nil
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.git_commit_sha
|
65
|
+
exec_git_command("git rev-parse HEAD")
|
66
|
+
rescue => e
|
67
|
+
log_failure(e, "git commit sha")
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.git_branch
|
72
|
+
exec_git_command("git rev-parse --abbrev-ref HEAD")
|
73
|
+
rescue => e
|
74
|
+
log_failure(e, "git branch")
|
75
|
+
nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def self.git_tag
|
79
|
+
exec_git_command("git tag --points-at HEAD")
|
80
|
+
rescue => e
|
81
|
+
log_failure(e, "git tag")
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.git_commit_message
|
86
|
+
exec_git_command("git show -s --format=%s")
|
87
|
+
rescue => e
|
88
|
+
log_failure(e, "git commit message")
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.git_commit_users
|
93
|
+
# Get committer and author information in one command.
|
94
|
+
output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
|
95
|
+
unless output
|
96
|
+
Datadog.logger.debug(
|
97
|
+
"Unable to read git commit users: git command output is nil"
|
98
|
+
)
|
99
|
+
nil_user = NilUser.new
|
100
|
+
return [nil_user, nil_user]
|
101
|
+
end
|
102
|
+
|
103
|
+
author_name, author_email, author_timestamp,
|
104
|
+
committer_name, committer_email, committer_timestamp = output.split("\t").each(&:strip!)
|
105
|
+
|
106
|
+
author = User.new(author_name, author_email, author_timestamp)
|
107
|
+
committer = User.new(committer_name, committer_email, committer_timestamp)
|
108
|
+
|
109
|
+
[author, committer]
|
110
|
+
rescue => e
|
111
|
+
log_failure(e, "git commit users")
|
112
|
+
|
113
|
+
nil_user = NilUser.new
|
114
|
+
[nil_user, nil_user]
|
115
|
+
end
|
116
|
+
|
117
|
+
# returns maximum of 1000 latest commits in the last month
|
118
|
+
def self.git_commits
|
119
|
+
output = exec_git_command("git log --format=%H -n 1000 --since=\"1 month ago\"")
|
120
|
+
return [] if output.nil?
|
121
|
+
|
122
|
+
output.split("\n")
|
123
|
+
rescue => e
|
124
|
+
log_failure(e, "git commits")
|
125
|
+
[]
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.git_commits_rev_list(included_commits:, excluded_commits:)
|
129
|
+
included_commits = filter_invalid_commits(included_commits).join(" ")
|
130
|
+
excluded_commits = filter_invalid_commits(excluded_commits).map! { |sha| "^#{sha}" }.join(" ")
|
131
|
+
|
132
|
+
exec_git_command(
|
133
|
+
"git rev-list " \
|
134
|
+
"--objects " \
|
135
|
+
"--no-object-names " \
|
136
|
+
"--filter=blob:none " \
|
137
|
+
"--since=\"1 month ago\" " \
|
138
|
+
"#{excluded_commits} #{included_commits}"
|
139
|
+
)
|
140
|
+
rescue => e
|
141
|
+
log_failure(e, "git commits rev list")
|
142
|
+
nil
|
143
|
+
end
|
144
|
+
|
145
|
+
def self.git_generate_packfiles(included_commits:, excluded_commits:, path:)
|
146
|
+
return nil unless File.exist?(path)
|
147
|
+
|
148
|
+
commit_tree = git_commits_rev_list(included_commits: included_commits, excluded_commits: excluded_commits)
|
149
|
+
return nil if commit_tree.nil?
|
150
|
+
|
151
|
+
basename = SecureRandom.hex(4)
|
152
|
+
|
153
|
+
exec_git_command(
|
154
|
+
"git pack-objects --compression=9 --max-pack-size=3m #{path}/#{basename}",
|
155
|
+
stdin: commit_tree
|
156
|
+
)
|
157
|
+
|
158
|
+
basename
|
159
|
+
rescue => e
|
160
|
+
log_failure(e, "git generate packfiles")
|
161
|
+
nil
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.git_shallow_clone?
|
165
|
+
exec_git_command("git rev-parse --is-shallow-repository") == "true"
|
166
|
+
rescue => e
|
167
|
+
log_failure(e, "git shallow clone")
|
168
|
+
false
|
169
|
+
end
|
170
|
+
|
171
|
+
def self.git_unshallow
|
172
|
+
exec_git_command(
|
173
|
+
"git fetch " \
|
174
|
+
"--shallow-since=\"1 month ago\" " \
|
175
|
+
"--update-shallow " \
|
176
|
+
"--filter=\"blob:none\" " \
|
177
|
+
"--recurse-submodules=no " \
|
178
|
+
"$(git config --default origin --get clone.defaultRemoteName) $(git rev-parse HEAD)"
|
179
|
+
)
|
180
|
+
rescue => e
|
181
|
+
log_failure(e, "git unshallow")
|
182
|
+
nil
|
183
|
+
end
|
184
|
+
|
185
|
+
# makes .exec_git_command private to make sure that this method
|
186
|
+
# is not called from outside of this module with insecure parameters
|
187
|
+
class << self
|
188
|
+
private
|
189
|
+
|
190
|
+
def filter_invalid_commits(commits)
|
191
|
+
commits.filter { |commit| Utils::Git.valid_commit_sha?(commit) }
|
192
|
+
end
|
193
|
+
|
194
|
+
def exec_git_command(cmd, stdin: nil)
|
195
|
+
# Shell injection is alleviated by making sure that no outside modules call this method.
|
196
|
+
# It is called only internally with static parameters.
|
197
|
+
# no-dd-sa:ruby-security/shell-injection
|
198
|
+
out, status = Open3.capture2e(cmd, stdin_data: stdin)
|
199
|
+
|
200
|
+
if status.nil?
|
201
|
+
retry_count = COMMAND_RETRY_COUNT
|
202
|
+
Datadog.logger.debug { "Opening pipe failed, starting retries..." }
|
203
|
+
while status.nil? && retry_count.positive?
|
204
|
+
# no-dd-sa:ruby-security/shell-injection
|
205
|
+
out, status = Open3.capture2e(cmd, stdin_data: stdin)
|
206
|
+
Datadog.logger.debug { "After retry status is [#{status}]" }
|
207
|
+
retry_count -= 1
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
if status.nil? || !status.success?
|
212
|
+
raise "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]"
|
213
|
+
end
|
214
|
+
|
215
|
+
# Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
|
216
|
+
# commit messages with UTF-8 characters like emojis
|
217
|
+
# We force output's encoding to be UTF-8 in this case
|
218
|
+
# This is safe to do as UTF-8 is compatible with US-ASCII
|
219
|
+
if Encoding.default_external == Encoding::US_ASCII
|
220
|
+
out = out.force_encoding(Encoding::UTF_8)
|
221
|
+
end
|
222
|
+
out.strip! # There's always a "\n" at the end of the command output
|
223
|
+
|
224
|
+
return nil if out.empty?
|
225
|
+
|
226
|
+
out
|
227
|
+
end
|
228
|
+
|
229
|
+
def log_failure(e, action)
|
230
|
+
Datadog.logger.debug(
|
231
|
+
"Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
|
232
|
+
)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "tmpdir"
|
4
|
+
require "fileutils"
|
5
|
+
|
6
|
+
require_relative "local_repository"
|
7
|
+
|
8
|
+
module Datadog
|
9
|
+
module CI
|
10
|
+
module Git
|
11
|
+
module Packfiles
|
12
|
+
def self.generate(included_commits:, excluded_commits:)
|
13
|
+
# @type var current_process_tmp_folder: String?
|
14
|
+
current_process_tmp_folder = nil
|
15
|
+
|
16
|
+
Dir.mktmpdir do |tmpdir|
|
17
|
+
prefix = LocalRepository.git_generate_packfiles(
|
18
|
+
included_commits: included_commits,
|
19
|
+
excluded_commits: excluded_commits,
|
20
|
+
path: tmpdir
|
21
|
+
)
|
22
|
+
|
23
|
+
if prefix.nil?
|
24
|
+
# git pack-files command fails if tmpdir is mounted on
|
25
|
+
# a different device from the current process directory
|
26
|
+
#
|
27
|
+
# @type var current_process_tmp_folder: String
|
28
|
+
current_process_tmp_folder = File.join(Dir.pwd, "tmp", "packfiles")
|
29
|
+
FileUtils.mkdir_p(current_process_tmp_folder)
|
30
|
+
|
31
|
+
prefix = LocalRepository.git_generate_packfiles(
|
32
|
+
included_commits: included_commits,
|
33
|
+
excluded_commits: excluded_commits,
|
34
|
+
path: current_process_tmp_folder
|
35
|
+
)
|
36
|
+
|
37
|
+
if prefix.nil?
|
38
|
+
Datadog.logger.debug("Packfiles generation failed twice, aborting")
|
39
|
+
break
|
40
|
+
end
|
41
|
+
|
42
|
+
tmpdir = current_process_tmp_folder
|
43
|
+
end
|
44
|
+
|
45
|
+
packfiles = Dir.entries(tmpdir) - %w[. ..]
|
46
|
+
if packfiles.empty?
|
47
|
+
Datadog.logger.debug("Empty packfiles list, aborting process")
|
48
|
+
break
|
49
|
+
end
|
50
|
+
|
51
|
+
packfiles.each do |packfile_name|
|
52
|
+
next unless packfile_name.start_with?(prefix)
|
53
|
+
next unless packfile_name.end_with?(".pack")
|
54
|
+
|
55
|
+
packfile_path = File.join(tmpdir, packfile_name)
|
56
|
+
|
57
|
+
yield packfile_path
|
58
|
+
end
|
59
|
+
end
|
60
|
+
rescue => e
|
61
|
+
Datadog.logger.debug("Packfiles could not be generated, error: #{e}")
|
62
|
+
ensure
|
63
|
+
if current_process_tmp_folder && File.exist?(current_process_tmp_folder)
|
64
|
+
FileUtils.remove_entry(current_process_tmp_folder)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "set"
|
5
|
+
|
6
|
+
require_relative "../ext/transport"
|
7
|
+
require_relative "../utils/git"
|
8
|
+
|
9
|
+
module Datadog
|
10
|
+
module CI
|
11
|
+
module Git
|
12
|
+
class SearchCommits
|
13
|
+
class ApiError < StandardError; end
|
14
|
+
|
15
|
+
attr_reader :api
|
16
|
+
|
17
|
+
def initialize(api:)
|
18
|
+
@api = api
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(repository_url, commits)
|
22
|
+
raise ApiError, "test visibility API is not configured" if api.nil?
|
23
|
+
|
24
|
+
http_response = api.api_request(
|
25
|
+
path: Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
|
26
|
+
payload: request_payload(repository_url, commits)
|
27
|
+
)
|
28
|
+
raise ApiError, "Failed to search commits: #{http_response.inspect}" unless http_response.ok?
|
29
|
+
|
30
|
+
response_payload = parse_json_response(http_response)
|
31
|
+
extract_commits(response_payload)
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
def request_payload(repository_url, commits)
|
37
|
+
{
|
38
|
+
meta: {
|
39
|
+
repository_url: repository_url
|
40
|
+
},
|
41
|
+
data: commits.filter_map do |commit|
|
42
|
+
next unless Utils::Git.valid_commit_sha?(commit)
|
43
|
+
|
44
|
+
{
|
45
|
+
id: commit,
|
46
|
+
type: "commit"
|
47
|
+
}
|
48
|
+
end
|
49
|
+
}.to_json
|
50
|
+
end
|
51
|
+
|
52
|
+
def parse_json_response(http_response)
|
53
|
+
JSON.parse(http_response.payload)
|
54
|
+
rescue JSON::ParserError => e
|
55
|
+
raise ApiError, "Failed to parse search commits response: #{e}. Payload was: #{http_response.payload}"
|
56
|
+
end
|
57
|
+
|
58
|
+
def extract_commits(response_payload)
|
59
|
+
result = Set.new
|
60
|
+
|
61
|
+
response_payload.fetch("data").each do |commit_json|
|
62
|
+
raise ApiError, "Invalid commit type response #{commit_json}" unless commit_json["type"] == "commit"
|
63
|
+
|
64
|
+
commit_sha = commit_json["id"]
|
65
|
+
raise ApiError, "Invalid commit SHA response #{commit_sha}" unless Utils::Git.valid_commit_sha?(commit_sha)
|
66
|
+
|
67
|
+
result.add(commit_sha)
|
68
|
+
end
|
69
|
+
|
70
|
+
result
|
71
|
+
rescue KeyError => e
|
72
|
+
raise ApiError, "Malformed search commits response: #{e}. Payload was: #{response_payload}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "tmpdir"
|
4
|
+
require "fileutils"
|
5
|
+
|
6
|
+
require_relative "local_repository"
|
7
|
+
require_relative "search_commits"
|
8
|
+
require_relative "upload_packfile"
|
9
|
+
require_relative "packfiles"
|
10
|
+
|
11
|
+
module Datadog
|
12
|
+
module CI
|
13
|
+
module Git
|
14
|
+
class TreeUploader
|
15
|
+
attr_reader :api
|
16
|
+
|
17
|
+
def initialize(api:)
|
18
|
+
@api = api
|
19
|
+
end
|
20
|
+
|
21
|
+
def call(repository_url)
|
22
|
+
if api.nil?
|
23
|
+
Datadog.logger.debug("API is not configured, aborting git upload")
|
24
|
+
return
|
25
|
+
end
|
26
|
+
|
27
|
+
Datadog.logger.debug { "Uploading git tree for repository #{repository_url}" }
|
28
|
+
|
29
|
+
latest_commits = LocalRepository.git_commits
|
30
|
+
head_commit = latest_commits&.first
|
31
|
+
if head_commit.nil?
|
32
|
+
Datadog.logger.debug("Got empty latest commits list, aborting git upload")
|
33
|
+
return
|
34
|
+
end
|
35
|
+
|
36
|
+
begin
|
37
|
+
# ask the backend for the list of commits it already has
|
38
|
+
known_commits, new_commits = fetch_known_commits_and_split(repository_url, latest_commits)
|
39
|
+
# if all commits are present in the backend, we don't need to upload anything
|
40
|
+
if new_commits.empty?
|
41
|
+
Datadog.logger.debug("No new commits to upload")
|
42
|
+
return
|
43
|
+
end
|
44
|
+
|
45
|
+
# quite often we deal with shallow clones in CI environment
|
46
|
+
if LocalRepository.git_shallow_clone? && LocalRepository.git_unshallow
|
47
|
+
Datadog.logger.debug("Detected shallow clone and unshallowed the repository, repeating commits search")
|
48
|
+
|
49
|
+
# re-run the search with the updated commit list after unshallowing
|
50
|
+
known_commits, new_commits = fetch_known_commits_and_split(
|
51
|
+
repository_url,
|
52
|
+
LocalRepository.git_commits
|
53
|
+
)
|
54
|
+
end
|
55
|
+
rescue SearchCommits::ApiError => e
|
56
|
+
Datadog.logger.debug("SearchCommits failed with #{e}, aborting git upload")
|
57
|
+
return
|
58
|
+
end
|
59
|
+
|
60
|
+
Datadog.logger.debug { "Uploading packfiles for commits: #{new_commits}" }
|
61
|
+
uploader = UploadPackfile.new(
|
62
|
+
api: api,
|
63
|
+
head_commit_sha: head_commit,
|
64
|
+
repository_url: repository_url
|
65
|
+
)
|
66
|
+
Packfiles.generate(included_commits: new_commits, excluded_commits: known_commits) do |filepath|
|
67
|
+
uploader.call(filepath: filepath)
|
68
|
+
rescue UploadPackfile::ApiError => e
|
69
|
+
Datadog.logger.debug("Packfile upload failed with #{e}")
|
70
|
+
break
|
71
|
+
end
|
72
|
+
ensure
|
73
|
+
Datadog.logger.debug("Git tree upload finished")
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
# Split the latest commits list into known and new commits
|
79
|
+
# based on the backend response provided by /search_commits endpoint
|
80
|
+
def fetch_known_commits_and_split(repository_url, latest_commits)
|
81
|
+
Datadog.logger.debug { "Checking the latest commits list with backend: #{latest_commits}" }
|
82
|
+
backend_commits = SearchCommits.new(api: api).call(repository_url, latest_commits)
|
83
|
+
latest_commits.partition do |commit|
|
84
|
+
backend_commits.include?(commit)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "securerandom"
|
5
|
+
|
6
|
+
module Datadog
|
7
|
+
module CI
|
8
|
+
module Git
|
9
|
+
class UploadPackfile
|
10
|
+
class ApiError < StandardError; end
|
11
|
+
|
12
|
+
attr_reader :api, :head_commit_sha, :repository_url
|
13
|
+
|
14
|
+
def initialize(api:, head_commit_sha:, repository_url:)
|
15
|
+
@api = api
|
16
|
+
@head_commit_sha = head_commit_sha
|
17
|
+
@repository_url = repository_url
|
18
|
+
end
|
19
|
+
|
20
|
+
def call(filepath:)
|
21
|
+
raise ApiError, "test visibility API is not configured" if api.nil?
|
22
|
+
|
23
|
+
payload_boundary = SecureRandom.uuid
|
24
|
+
|
25
|
+
filename = File.basename(filepath)
|
26
|
+
packfile_contents = read_file(filepath)
|
27
|
+
|
28
|
+
payload = request_payload(payload_boundary, filename, packfile_contents)
|
29
|
+
content_type = "#{Ext::Transport::CONTENT_TYPE_MULTIPART_FORM_DATA}; boundary=#{payload_boundary}"
|
30
|
+
|
31
|
+
http_response = api.api_request(
|
32
|
+
path: Ext::Transport::DD_API_GIT_UPLOAD_PACKFILE_PATH,
|
33
|
+
payload: payload,
|
34
|
+
headers: {Ext::Transport::HEADER_CONTENT_TYPE => content_type}
|
35
|
+
)
|
36
|
+
|
37
|
+
raise ApiError, "Failed to upload packfile: #{http_response.inspect}" unless http_response.ok?
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def request_payload(boundary, filename, packfile_contents)
|
43
|
+
[
|
44
|
+
"--#{boundary}",
|
45
|
+
'Content-Disposition: form-data; name="pushedSha"',
|
46
|
+
"Content-Type: application/json",
|
47
|
+
"",
|
48
|
+
{data: {id: head_commit_sha, type: "commit"}, meta: {repository_url: repository_url}}.to_json,
|
49
|
+
"--#{boundary}",
|
50
|
+
"Content-Disposition: form-data; name=\"packfile\"; filename=\"#{filename}\"",
|
51
|
+
"Content-Type: application/octet-stream",
|
52
|
+
"",
|
53
|
+
packfile_contents,
|
54
|
+
"--#{boundary}--"
|
55
|
+
].join("\r\n")
|
56
|
+
end
|
57
|
+
|
58
|
+
def read_file(filepath)
|
59
|
+
File.read(filepath)
|
60
|
+
rescue => e
|
61
|
+
raise ApiError, "Failed to read packfile: #{e.message}"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module CI
|
5
|
+
module Git
|
6
|
+
class User
|
7
|
+
attr_reader :name, :email, :timestamp
|
8
|
+
|
9
|
+
def initialize(name, email, timestamp)
|
10
|
+
@name = name
|
11
|
+
@email = email
|
12
|
+
@timestamp = timestamp
|
13
|
+
end
|
14
|
+
|
15
|
+
def date
|
16
|
+
return nil if timestamp.nil?
|
17
|
+
|
18
|
+
Time.at(timestamp.to_i).utc.to_datetime.iso8601
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class NilUser < User
|
23
|
+
def initialize
|
24
|
+
super(nil, nil, nil)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|