datadog-ci 1.0.0.beta1 → 1.0.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +105 -60
  3. data/README.md +37 -46
  4. data/lib/datadog/ci/configuration/components.rb +51 -9
  5. data/lib/datadog/ci/configuration/settings.rb +6 -0
  6. data/lib/datadog/ci/contrib/cucumber/formatter.rb +10 -5
  7. data/lib/datadog/ci/contrib/cucumber/patcher.rb +3 -0
  8. data/lib/datadog/ci/contrib/cucumber/step.rb +27 -0
  9. data/lib/datadog/ci/contrib/minitest/patcher.rb +2 -2
  10. data/lib/datadog/ci/contrib/minitest/test.rb +105 -0
  11. data/lib/datadog/ci/contrib/rspec/example.rb +11 -5
  12. data/lib/datadog/ci/ext/environment/providers/local_git.rb +8 -79
  13. data/lib/datadog/ci/ext/environment.rb +11 -16
  14. data/lib/datadog/ci/ext/settings.rb +1 -0
  15. data/lib/datadog/ci/ext/test.rb +8 -1
  16. data/lib/datadog/ci/ext/transport.rb +8 -0
  17. data/lib/datadog/ci/git/local_repository.rb +238 -0
  18. data/lib/datadog/ci/git/packfiles.rb +70 -0
  19. data/lib/datadog/ci/git/search_commits.rb +77 -0
  20. data/lib/datadog/ci/git/tree_uploader.rb +90 -0
  21. data/lib/datadog/ci/git/upload_packfile.rb +66 -0
  22. data/lib/datadog/ci/git/user.rb +29 -0
  23. data/lib/datadog/ci/itr/coverage/event.rb +18 -1
  24. data/lib/datadog/ci/itr/coverage/writer.rb +114 -0
  25. data/lib/datadog/ci/itr/runner.rb +134 -11
  26. data/lib/datadog/ci/itr/skippable.rb +108 -0
  27. data/lib/datadog/ci/span.rb +16 -0
  28. data/lib/datadog/ci/test.rb +37 -12
  29. data/lib/datadog/ci/test_module.rb +2 -2
  30. data/lib/datadog/ci/test_session.rb +2 -2
  31. data/lib/datadog/ci/test_suite.rb +2 -2
  32. data/lib/datadog/ci/test_visibility/null_recorder.rb +4 -1
  33. data/lib/datadog/ci/test_visibility/recorder.rb +47 -9
  34. data/lib/datadog/ci/test_visibility/serializers/base.rb +3 -2
  35. data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb +3 -3
  36. data/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb +6 -6
  37. data/lib/datadog/ci/test_visibility/serializers/test_v2.rb +14 -2
  38. data/lib/datadog/ci/test_visibility/transport.rb +6 -2
  39. data/lib/datadog/ci/transport/http.rb +24 -4
  40. data/lib/datadog/ci/transport/remote_settings_api.rb +14 -6
  41. data/lib/datadog/ci/utils/configuration.rb +2 -2
  42. data/lib/datadog/ci/utils/git.rb +6 -67
  43. data/lib/datadog/ci/utils/parsing.rb +16 -0
  44. data/lib/datadog/ci/utils/test_run.rb +25 -0
  45. data/lib/datadog/ci/version.rb +1 -1
  46. data/lib/datadog/ci/worker.rb +35 -0
  47. data/lib/datadog/ci.rb +4 -0
  48. metadata +17 -6
  49. data/lib/datadog/ci/contrib/minitest/hooks.rb +0 -75
@@ -0,0 +1,238 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open3"
4
+ require "pathname"
5
+
6
+ require_relative "user"
7
+
8
+ module Datadog
9
+ module CI
10
+ module Git
11
+ module LocalRepository
12
+ COMMAND_RETRY_COUNT = 3
13
+
14
+ def self.root
15
+ return @root if defined?(@root)
16
+
17
+ @root = git_root || Dir.pwd
18
+ end
19
+
20
+ def self.relative_to_root(path)
21
+ return "" if path.nil?
22
+
23
+ root_path = root
24
+ return path if root_path.nil?
25
+
26
+ path = Pathname.new(File.expand_path(path))
27
+ root_path = Pathname.new(root_path)
28
+
29
+ path.relative_path_from(root_path).to_s
30
+ end
31
+
32
+ def self.repository_name
33
+ return @repository_name if defined?(@repository_name)
34
+
35
+ git_remote_url = git_repository_url
36
+
37
+ # return git repository name from remote url without .git extension
38
+ last_path_segment = git_remote_url.split("/").last if git_remote_url
39
+ @repository_name = last_path_segment.gsub(".git", "") if last_path_segment
40
+ @repository_name ||= current_folder_name
41
+ rescue => e
42
+ log_failure(e, "git repository name")
43
+ @repository_name = current_folder_name
44
+ end
45
+
46
+ def self.current_folder_name
47
+ File.basename(root)
48
+ end
49
+
50
+ def self.git_repository_url
51
+ exec_git_command("git ls-remote --get-url")
52
+ rescue => e
53
+ log_failure(e, "git repository url")
54
+ nil
55
+ end
56
+
57
+ def self.git_root
58
+ exec_git_command("git rev-parse --show-toplevel")
59
+ rescue => e
60
+ log_failure(e, "git root path")
61
+ nil
62
+ end
63
+
64
+ def self.git_commit_sha
65
+ exec_git_command("git rev-parse HEAD")
66
+ rescue => e
67
+ log_failure(e, "git commit sha")
68
+ nil
69
+ end
70
+
71
+ def self.git_branch
72
+ exec_git_command("git rev-parse --abbrev-ref HEAD")
73
+ rescue => e
74
+ log_failure(e, "git branch")
75
+ nil
76
+ end
77
+
78
+ def self.git_tag
79
+ exec_git_command("git tag --points-at HEAD")
80
+ rescue => e
81
+ log_failure(e, "git tag")
82
+ nil
83
+ end
84
+
85
+ def self.git_commit_message
86
+ exec_git_command("git show -s --format=%s")
87
+ rescue => e
88
+ log_failure(e, "git commit message")
89
+ nil
90
+ end
91
+
92
+ def self.git_commit_users
93
+ # Get committer and author information in one command.
94
+ output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
95
+ unless output
96
+ Datadog.logger.debug(
97
+ "Unable to read git commit users: git command output is nil"
98
+ )
99
+ nil_user = NilUser.new
100
+ return [nil_user, nil_user]
101
+ end
102
+
103
+ author_name, author_email, author_timestamp,
104
+ committer_name, committer_email, committer_timestamp = output.split("\t").each(&:strip!)
105
+
106
+ author = User.new(author_name, author_email, author_timestamp)
107
+ committer = User.new(committer_name, committer_email, committer_timestamp)
108
+
109
+ [author, committer]
110
+ rescue => e
111
+ log_failure(e, "git commit users")
112
+
113
+ nil_user = NilUser.new
114
+ [nil_user, nil_user]
115
+ end
116
+
117
+ # returns maximum of 1000 latest commits in the last month
118
+ def self.git_commits
119
+ output = exec_git_command("git log --format=%H -n 1000 --since=\"1 month ago\"")
120
+ return [] if output.nil?
121
+
122
+ output.split("\n")
123
+ rescue => e
124
+ log_failure(e, "git commits")
125
+ []
126
+ end
127
+
128
+ def self.git_commits_rev_list(included_commits:, excluded_commits:)
129
+ included_commits = filter_invalid_commits(included_commits).join(" ")
130
+ excluded_commits = filter_invalid_commits(excluded_commits).map! { |sha| "^#{sha}" }.join(" ")
131
+
132
+ exec_git_command(
133
+ "git rev-list " \
134
+ "--objects " \
135
+ "--no-object-names " \
136
+ "--filter=blob:none " \
137
+ "--since=\"1 month ago\" " \
138
+ "#{excluded_commits} #{included_commits}"
139
+ )
140
+ rescue => e
141
+ log_failure(e, "git commits rev list")
142
+ nil
143
+ end
144
+
145
+ def self.git_generate_packfiles(included_commits:, excluded_commits:, path:)
146
+ return nil unless File.exist?(path)
147
+
148
+ commit_tree = git_commits_rev_list(included_commits: included_commits, excluded_commits: excluded_commits)
149
+ return nil if commit_tree.nil?
150
+
151
+ basename = SecureRandom.hex(4)
152
+
153
+ exec_git_command(
154
+ "git pack-objects --compression=9 --max-pack-size=3m #{path}/#{basename}",
155
+ stdin: commit_tree
156
+ )
157
+
158
+ basename
159
+ rescue => e
160
+ log_failure(e, "git generate packfiles")
161
+ nil
162
+ end
163
+
164
+ def self.git_shallow_clone?
165
+ exec_git_command("git rev-parse --is-shallow-repository") == "true"
166
+ rescue => e
167
+ log_failure(e, "git shallow clone")
168
+ false
169
+ end
170
+
171
+ def self.git_unshallow
172
+ exec_git_command(
173
+ "git fetch " \
174
+ "--shallow-since=\"1 month ago\" " \
175
+ "--update-shallow " \
176
+ "--filter=\"blob:none\" " \
177
+ "--recurse-submodules=no " \
178
+ "$(git config --default origin --get clone.defaultRemoteName) $(git rev-parse HEAD)"
179
+ )
180
+ rescue => e
181
+ log_failure(e, "git unshallow")
182
+ nil
183
+ end
184
+
185
+ # makes .exec_git_command private to make sure that this method
186
+ # is not called from outside of this module with insecure parameters
187
+ class << self
188
+ private
189
+
190
+ def filter_invalid_commits(commits)
191
+ commits.filter { |commit| Utils::Git.valid_commit_sha?(commit) }
192
+ end
193
+
194
+ def exec_git_command(cmd, stdin: nil)
195
+ # Shell injection is alleviated by making sure that no outside modules call this method.
196
+ # It is called only internally with static parameters.
197
+ # no-dd-sa:ruby-security/shell-injection
198
+ out, status = Open3.capture2e(cmd, stdin_data: stdin)
199
+
200
+ if status.nil?
201
+ retry_count = COMMAND_RETRY_COUNT
202
+ Datadog.logger.debug { "Opening pipe failed, starting retries..." }
203
+ while status.nil? && retry_count.positive?
204
+ # no-dd-sa:ruby-security/shell-injection
205
+ out, status = Open3.capture2e(cmd, stdin_data: stdin)
206
+ Datadog.logger.debug { "After retry status is [#{status}]" }
207
+ retry_count -= 1
208
+ end
209
+ end
210
+
211
+ if status.nil? || !status.success?
212
+ raise "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]"
213
+ end
214
+
215
+ # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
216
+ # commit messages with UTF-8 characters like emojis
217
+ # We force output's encoding to be UTF-8 in this case
218
+ # This is safe to do as UTF-8 is compatible with US-ASCII
219
+ if Encoding.default_external == Encoding::US_ASCII
220
+ out = out.force_encoding(Encoding::UTF_8)
221
+ end
222
+ out.strip! # There's always a "\n" at the end of the command output
223
+
224
+ return nil if out.empty?
225
+
226
+ out
227
+ end
228
+
229
+ def log_failure(e, action)
230
+ Datadog.logger.debug(
231
+ "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
232
+ )
233
+ end
234
+ end
235
+ end
236
+ end
237
+ end
238
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tmpdir"
4
+ require "fileutils"
5
+
6
+ require_relative "local_repository"
7
+
8
+ module Datadog
9
+ module CI
10
+ module Git
11
+ module Packfiles
12
+ def self.generate(included_commits:, excluded_commits:)
13
+ # @type var current_process_tmp_folder: String?
14
+ current_process_tmp_folder = nil
15
+
16
+ Dir.mktmpdir do |tmpdir|
17
+ prefix = LocalRepository.git_generate_packfiles(
18
+ included_commits: included_commits,
19
+ excluded_commits: excluded_commits,
20
+ path: tmpdir
21
+ )
22
+
23
+ if prefix.nil?
24
+ # git pack-files command fails if tmpdir is mounted on
25
+ # a different device from the current process directory
26
+ #
27
+ # @type var current_process_tmp_folder: String
28
+ current_process_tmp_folder = File.join(Dir.pwd, "tmp", "packfiles")
29
+ FileUtils.mkdir_p(current_process_tmp_folder)
30
+
31
+ prefix = LocalRepository.git_generate_packfiles(
32
+ included_commits: included_commits,
33
+ excluded_commits: excluded_commits,
34
+ path: current_process_tmp_folder
35
+ )
36
+
37
+ if prefix.nil?
38
+ Datadog.logger.debug("Packfiles generation failed twice, aborting")
39
+ break
40
+ end
41
+
42
+ tmpdir = current_process_tmp_folder
43
+ end
44
+
45
+ packfiles = Dir.entries(tmpdir) - %w[. ..]
46
+ if packfiles.empty?
47
+ Datadog.logger.debug("Empty packfiles list, aborting process")
48
+ break
49
+ end
50
+
51
+ packfiles.each do |packfile_name|
52
+ next unless packfile_name.start_with?(prefix)
53
+ next unless packfile_name.end_with?(".pack")
54
+
55
+ packfile_path = File.join(tmpdir, packfile_name)
56
+
57
+ yield packfile_path
58
+ end
59
+ end
60
+ rescue => e
61
+ Datadog.logger.debug("Packfiles could not be generated, error: #{e}")
62
+ ensure
63
+ if current_process_tmp_folder && File.exist?(current_process_tmp_folder)
64
+ FileUtils.remove_entry(current_process_tmp_folder)
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "set"
5
+
6
+ require_relative "../ext/transport"
7
+ require_relative "../utils/git"
8
+
9
+ module Datadog
10
+ module CI
11
+ module Git
12
+ class SearchCommits
13
+ class ApiError < StandardError; end
14
+
15
+ attr_reader :api
16
+
17
+ def initialize(api:)
18
+ @api = api
19
+ end
20
+
21
+ def call(repository_url, commits)
22
+ raise ApiError, "test visibility API is not configured" if api.nil?
23
+
24
+ http_response = api.api_request(
25
+ path: Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
26
+ payload: request_payload(repository_url, commits)
27
+ )
28
+ raise ApiError, "Failed to search commits: #{http_response.inspect}" unless http_response.ok?
29
+
30
+ response_payload = parse_json_response(http_response)
31
+ extract_commits(response_payload)
32
+ end
33
+
34
+ private
35
+
36
+ def request_payload(repository_url, commits)
37
+ {
38
+ meta: {
39
+ repository_url: repository_url
40
+ },
41
+ data: commits.filter_map do |commit|
42
+ next unless Utils::Git.valid_commit_sha?(commit)
43
+
44
+ {
45
+ id: commit,
46
+ type: "commit"
47
+ }
48
+ end
49
+ }.to_json
50
+ end
51
+
52
+ def parse_json_response(http_response)
53
+ JSON.parse(http_response.payload)
54
+ rescue JSON::ParserError => e
55
+ raise ApiError, "Failed to parse search commits response: #{e}. Payload was: #{http_response.payload}"
56
+ end
57
+
58
+ def extract_commits(response_payload)
59
+ result = Set.new
60
+
61
+ response_payload.fetch("data").each do |commit_json|
62
+ raise ApiError, "Invalid commit type response #{commit_json}" unless commit_json["type"] == "commit"
63
+
64
+ commit_sha = commit_json["id"]
65
+ raise ApiError, "Invalid commit SHA response #{commit_sha}" unless Utils::Git.valid_commit_sha?(commit_sha)
66
+
67
+ result.add(commit_sha)
68
+ end
69
+
70
+ result
71
+ rescue KeyError => e
72
+ raise ApiError, "Malformed search commits response: #{e}. Payload was: #{response_payload}"
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,90 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tmpdir"
4
+ require "fileutils"
5
+
6
+ require_relative "local_repository"
7
+ require_relative "search_commits"
8
+ require_relative "upload_packfile"
9
+ require_relative "packfiles"
10
+
11
+ module Datadog
12
+ module CI
13
+ module Git
14
+ class TreeUploader
15
+ attr_reader :api
16
+
17
+ def initialize(api:)
18
+ @api = api
19
+ end
20
+
21
+ def call(repository_url)
22
+ if api.nil?
23
+ Datadog.logger.debug("API is not configured, aborting git upload")
24
+ return
25
+ end
26
+
27
+ Datadog.logger.debug { "Uploading git tree for repository #{repository_url}" }
28
+
29
+ latest_commits = LocalRepository.git_commits
30
+ head_commit = latest_commits&.first
31
+ if head_commit.nil?
32
+ Datadog.logger.debug("Got empty latest commits list, aborting git upload")
33
+ return
34
+ end
35
+
36
+ begin
37
+ # ask the backend for the list of commits it already has
38
+ known_commits, new_commits = fetch_known_commits_and_split(repository_url, latest_commits)
39
+ # if all commits are present in the backend, we don't need to upload anything
40
+ if new_commits.empty?
41
+ Datadog.logger.debug("No new commits to upload")
42
+ return
43
+ end
44
+
45
+ # quite often we deal with shallow clones in CI environment
46
+ if LocalRepository.git_shallow_clone? && LocalRepository.git_unshallow
47
+ Datadog.logger.debug("Detected shallow clone and unshallowed the repository, repeating commits search")
48
+
49
+ # re-run the search with the updated commit list after unshallowing
50
+ known_commits, new_commits = fetch_known_commits_and_split(
51
+ repository_url,
52
+ LocalRepository.git_commits
53
+ )
54
+ end
55
+ rescue SearchCommits::ApiError => e
56
+ Datadog.logger.debug("SearchCommits failed with #{e}, aborting git upload")
57
+ return
58
+ end
59
+
60
+ Datadog.logger.debug { "Uploading packfiles for commits: #{new_commits}" }
61
+ uploader = UploadPackfile.new(
62
+ api: api,
63
+ head_commit_sha: head_commit,
64
+ repository_url: repository_url
65
+ )
66
+ Packfiles.generate(included_commits: new_commits, excluded_commits: known_commits) do |filepath|
67
+ uploader.call(filepath: filepath)
68
+ rescue UploadPackfile::ApiError => e
69
+ Datadog.logger.debug("Packfile upload failed with #{e}")
70
+ break
71
+ end
72
+ ensure
73
+ Datadog.logger.debug("Git tree upload finished")
74
+ end
75
+
76
+ private
77
+
78
+ # Split the latest commits list into known and new commits
79
+ # based on the backend response provided by /search_commits endpoint
80
+ def fetch_known_commits_and_split(repository_url, latest_commits)
81
+ Datadog.logger.debug { "Checking the latest commits list with backend: #{latest_commits}" }
82
+ backend_commits = SearchCommits.new(api: api).call(repository_url, latest_commits)
83
+ latest_commits.partition do |commit|
84
+ backend_commits.include?(commit)
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "securerandom"
5
+
6
+ module Datadog
7
+ module CI
8
+ module Git
9
+ class UploadPackfile
10
+ class ApiError < StandardError; end
11
+
12
+ attr_reader :api, :head_commit_sha, :repository_url
13
+
14
+ def initialize(api:, head_commit_sha:, repository_url:)
15
+ @api = api
16
+ @head_commit_sha = head_commit_sha
17
+ @repository_url = repository_url
18
+ end
19
+
20
+ def call(filepath:)
21
+ raise ApiError, "test visibility API is not configured" if api.nil?
22
+
23
+ payload_boundary = SecureRandom.uuid
24
+
25
+ filename = File.basename(filepath)
26
+ packfile_contents = read_file(filepath)
27
+
28
+ payload = request_payload(payload_boundary, filename, packfile_contents)
29
+ content_type = "#{Ext::Transport::CONTENT_TYPE_MULTIPART_FORM_DATA}; boundary=#{payload_boundary}"
30
+
31
+ http_response = api.api_request(
32
+ path: Ext::Transport::DD_API_GIT_UPLOAD_PACKFILE_PATH,
33
+ payload: payload,
34
+ headers: {Ext::Transport::HEADER_CONTENT_TYPE => content_type}
35
+ )
36
+
37
+ raise ApiError, "Failed to upload packfile: #{http_response.inspect}" unless http_response.ok?
38
+ end
39
+
40
+ private
41
+
42
+ def request_payload(boundary, filename, packfile_contents)
43
+ [
44
+ "--#{boundary}",
45
+ 'Content-Disposition: form-data; name="pushedSha"',
46
+ "Content-Type: application/json",
47
+ "",
48
+ {data: {id: head_commit_sha, type: "commit"}, meta: {repository_url: repository_url}}.to_json,
49
+ "--#{boundary}",
50
+ "Content-Disposition: form-data; name=\"packfile\"; filename=\"#{filename}\"",
51
+ "Content-Type: application/octet-stream",
52
+ "",
53
+ packfile_contents,
54
+ "--#{boundary}--"
55
+ ].join("\r\n")
56
+ end
57
+
58
+ def read_file(filepath)
59
+ File.read(filepath)
60
+ rescue => e
61
+ raise ApiError, "Failed to read packfile: #{e.message}"
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module CI
5
+ module Git
6
+ class User
7
+ attr_reader :name, :email, :timestamp
8
+
9
+ def initialize(name, email, timestamp)
10
+ @name = name
11
+ @email = email
12
+ @timestamp = timestamp
13
+ end
14
+
15
+ def date
16
+ return nil if timestamp.nil?
17
+
18
+ Time.at(timestamp.to_i).utc.to_datetime.iso8601
19
+ end
20
+ end
21
+
22
+ class NilUser < User
23
+ def initialize
24
+ super(nil, nil, nil)
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -2,6 +2,8 @@
2
2
 
3
3
  require "msgpack"
4
4
 
5
+ require_relative "../../git/local_repository"
6
+
5
7
  module Datadog
6
8
  module CI
7
9
  module ITR
@@ -50,13 +52,28 @@ module Datadog
50
52
  files.each do |filename|
51
53
  packer.write_map_header(1)
52
54
  packer.write("filename")
53
- packer.write(filename)
55
+ packer.write(Git::LocalRepository.relative_to_root(filename))
54
56
  end
55
57
  end
56
58
 
57
59
  def to_s
58
60
  "Coverage::Event[test_id=#{test_id}, test_suite_id=#{test_suite_id}, test_session_id=#{test_session_id}, coverage=#{coverage}]"
59
61
  end
62
+
63
+ # Return a human readable version of the event
64
+ def pretty_print(q)
65
+ q.group 0 do
66
+ q.breakable
67
+ q.text "Test ID: #{@test_id}\n"
68
+ q.text "Test Suite ID: #{@test_suite_id}\n"
69
+ q.text "Test Session ID: #{@test_session_id}\n"
70
+ q.group(2, "Files: [", "]\n") do
71
+ q.seplist @coverage.keys.each do |key|
72
+ q.text key
73
+ end
74
+ end
75
+ end
76
+ end
60
77
  end
61
78
  end
62
79
  end