riptide 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/LICENSE.txt +21 -0
- data/README.md +157 -0
- data/exe/riptide +6 -0
- data/ext/riptide/extconf.rb +49 -0
- data/ext/riptide/riptide_native.c +892 -0
- data/ext/riptide/ruby_source.rb +80 -0
- data/lib/minitest/riptide_plugin.rb +20 -0
- data/lib/riptide/analysis/fixture.rb +21 -0
- data/lib/riptide/analysis/ruby.rb +208 -0
- data/lib/riptide/analysis/schema.rb +60 -0
- data/lib/riptide/analysis/sql.rb +83 -0
- data/lib/riptide/analyzer.rb +138 -0
- data/lib/riptide/assertions.rb +82 -0
- data/lib/riptide/audit.rb +433 -0
- data/lib/riptide/change.rb +41 -0
- data/lib/riptide/cli.rb +109 -0
- data/lib/riptide/collectors.rb +386 -0
- data/lib/riptide/configuration.rb +63 -0
- data/lib/riptide/diff.rb +54 -0
- data/lib/riptide/env.rb +61 -0
- data/lib/riptide/git.rb +193 -0
- data/lib/riptide/harvest.rb +87 -0
- data/lib/riptide/index.rb +458 -0
- data/lib/riptide/inputs.rb +273 -0
- data/lib/riptide/layout.rb +45 -0
- data/lib/riptide/recorder.rb +374 -0
- data/lib/riptide/resolver.rb +67 -0
- data/lib/riptide/runner.rb +21 -0
- data/lib/riptide/scope.rb +39 -0
- data/lib/riptide/selector.rb +209 -0
- data/lib/riptide/session.rb +74 -0
- data/lib/riptide/source.rb +449 -0
- data/lib/riptide/store.rb +446 -0
- data/lib/riptide/tracer.rb +176 -0
- data/lib/riptide/version.rb +9 -0
- data/lib/riptide.rb +69 -0
- data/riptide.gemspec +34 -0
- metadata +116 -0
data/lib/riptide/git.rb
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "open3"
|
|
5
|
+
require "set"
|
|
6
|
+
require "tempfile"
|
|
7
|
+
|
|
8
|
+
module Riptide
|
|
9
|
+
# The git plumbing the store needs, run against one repository. Objects
|
|
10
|
+
# are read through one long-lived `git cat-file --batch`.
|
|
11
|
+
#
|
|
12
|
+
# Every command runs in its own process group under a deadline, so a
|
|
13
|
+
# stalled call can't hold up a build: the group gets TERM, which makes
|
|
14
|
+
# git drop its locks, then KILL. The batch reader's deadline is per
|
|
15
|
+
# object. Nothing prompts, nothing starts a gc while tests read .git, and
|
|
16
|
+
# an aborting git leaves no core file.
|
|
17
|
+
class Git
|
|
18
|
+
DEADLINE = 120
|
|
19
|
+
GRACE = 5
|
|
20
|
+
CONFIG = %w[-c gc.auto=0 -c maintenance.auto=false].freeze
|
|
21
|
+
ENV_DEFAULTS = { "GIT_TERMINAL_PROMPT" => "0", "GIT_SSH_COMMAND" => "ssh -o BatchMode=yes -o ConnectTimeout=15",
|
|
22
|
+
"GIT_HTTP_LOW_SPEED_LIMIT" => "1000", "GIT_HTTP_LOW_SPEED_TIME" => "30" }.freeze
|
|
23
|
+
|
|
24
|
+
def initialize(root, deadline: DEADLINE)
|
|
25
|
+
@root = root
|
|
26
|
+
@deadline = deadline
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# [stdout, exit status], where a signal exit is nil. Raises unless the
|
|
30
|
+
# status is in +allow+.
|
|
31
|
+
def run(*args, input: nil, env: {}, allow: [0], deadline: @deadline)
|
|
32
|
+
out, err, status = spawn(args, input, env, deadline)
|
|
33
|
+
raise Error, "git #{args.first} failed: #{err.strip}" unless allow.include?(status)
|
|
34
|
+
|
|
35
|
+
[out, status]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def output(*args, **options) = run(*args, **options).first
|
|
39
|
+
|
|
40
|
+
# A blob's content, or nil when the repository doesn't have it.
|
|
41
|
+
def read(sha)
|
|
42
|
+
batch_in, batch_out, = batch
|
|
43
|
+
batch_in.puts(sha)
|
|
44
|
+
header = batch_out.gets
|
|
45
|
+
raise IOError, "the reader exited" if header.nil?
|
|
46
|
+
return nil if header.end_with?("missing\n")
|
|
47
|
+
|
|
48
|
+
size = header.split.last.to_i
|
|
49
|
+
content = batch_out.read(size)
|
|
50
|
+
batch_out.read(1)
|
|
51
|
+
content
|
|
52
|
+
rescue IO::TimeoutError, IOError, SystemCallError => e
|
|
53
|
+
# A stalled or dead reader starts over on the next read.
|
|
54
|
+
if @batch
|
|
55
|
+
stdin, stdout, thread = @batch
|
|
56
|
+
@batch = nil
|
|
57
|
+
[stdin, stdout].each { |io| io.close unless io.closed? }
|
|
58
|
+
stop(thread)
|
|
59
|
+
end
|
|
60
|
+
raise Error, e.is_a?(IO::TimeoutError) ? "git cat-file timed out after #{@deadline}s" : "git cat-file failed: #{e.message}"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Every path under the root git tracks or would track, relative to the
|
|
64
|
+
# root: committed, staged, or untracked and not ignored.
|
|
65
|
+
def files
|
|
66
|
+
output("ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0").to_set
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Writes each of +blobs+ ({ path => content }) in one fast-import and
|
|
70
|
+
# returns { path => blob sha }. Ones already in +known+ aren't written
|
|
71
|
+
# again.
|
|
72
|
+
def write_blobs(blobs, known: Set.new)
|
|
73
|
+
shas = blobs.transform_values { |content| Digest::SHA1.hexdigest("blob #{content.bytesize}\0#{content}") }
|
|
74
|
+
fresh = blobs.reject { |path, _| known.include?(shas[path]) }.values.uniq
|
|
75
|
+
unless fresh.empty?
|
|
76
|
+
stream = fresh.map { |content| "blob\ndata #{content.bytesize}\n#{content}\n" }.join << "done\n"
|
|
77
|
+
run("fast-import", "--quiet", "--done", input: stream)
|
|
78
|
+
end
|
|
79
|
+
shas
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# { ref => object } for this repository's refs under +prefix+.
|
|
83
|
+
def refs(prefix)
|
|
84
|
+
output("for-each-ref", "--format=%(refname) %(objectname)", prefix).lines.to_h { |line| line.split.first(2) }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Deletes +names+ in one transaction.
|
|
88
|
+
def delete_refs(names)
|
|
89
|
+
run("update-ref", "--stdin", input: names.map { |name| "delete #{name}\n" }.join) unless names.empty?
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def merge_base(one, other)
|
|
93
|
+
out, status = run("merge-base", one, other, allow: [0, 1, 128])
|
|
94
|
+
status.zero? ? out.strip : nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The tree from merging +theirs+ into +ours+ as if +base+ were their
|
|
98
|
+
# common ancestor, or nil when it conflicts. All three can be trees.
|
|
99
|
+
def merge_tree(base, ours, theirs)
|
|
100
|
+
out, status = run("merge-tree", "--write-tree", "--no-messages", "--merge-base=#{base}", ours, theirs, allow: [0, 1])
|
|
101
|
+
status.zero? ? out.lines.first.strip : nil
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The object +rev+ names, or nil.
|
|
105
|
+
def resolve(rev)
|
|
106
|
+
out, status = run("rev-parse", "--verify", "--quiet", rev, allow: [0, 1, 128])
|
|
107
|
+
status.zero? ? out.strip : nil
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# { path => [mode, object] } for every file in +tree+, recursively, with
|
|
111
|
+
# paths from the top of the tree. Without --full-tree, git would list
|
|
112
|
+
# only what's under the root when the root is a subdirectory.
|
|
113
|
+
def entries(tree)
|
|
114
|
+
return {} if tree.nil?
|
|
115
|
+
|
|
116
|
+
output("ls-tree", "-r", "-z", "--full-tree", tree).split("\0").to_h do |entry|
|
|
117
|
+
info, path = entry.split("\t", 2)
|
|
118
|
+
mode, _, object = info.split
|
|
119
|
+
[path, [mode, object]]
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# { path => blob sha } for every file in a tree.
|
|
124
|
+
def tree(object) = entries(object).transform_values(&:last)
|
|
125
|
+
|
|
126
|
+
# The root's tree in the checked-out commit, whose paths are the ones
|
|
127
|
+
# riptide records: relative to the root, which can be a subdirectory of
|
|
128
|
+
# the repository.
|
|
129
|
+
def head_tree = "HEAD:#{@prefix ||= output("rev-parse", "--show-prefix").strip}"
|
|
130
|
+
|
|
131
|
+
# A tree holding +files+ ({ path => sha } or { path => [mode, sha] }),
|
|
132
|
+
# built in a throwaway index so the repository's own is never touched.
|
|
133
|
+
def write_tree(files)
|
|
134
|
+
Tempfile.create("riptide-index") do |index|
|
|
135
|
+
File.delete(index.path)
|
|
136
|
+
env = { "GIT_INDEX_FILE" => index.path }
|
|
137
|
+
listing = files.sort.map do |path, entry|
|
|
138
|
+
mode, sha = entry.is_a?(Array) ? entry : ["100644", entry]
|
|
139
|
+
"#{mode} #{sha}\t#{path}\0"
|
|
140
|
+
end.join
|
|
141
|
+
run("update-index", "-z", "--add", "--index-info", input: listing, env: env) unless listing.empty?
|
|
142
|
+
output("write-tree", "--missing-ok", env: env).strip
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# An annotated tag wrapping +tree+, dated now by riptide's own clock.
|
|
147
|
+
def tag_tree(tree, name)
|
|
148
|
+
content = "object #{tree}\ntype tree\ntag #{name}\ntagger riptide <riptide@users.noreply.github.com> #{Time.now.to_i} +0000\n\n#{name}\n"
|
|
149
|
+
output("mktag", input: content).strip
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
private
|
|
153
|
+
|
|
154
|
+
def spawn(args, input, env, deadline)
|
|
155
|
+
Open3.popen3(ENV_DEFAULTS.reject { |name, _| ENV.key?(name) }.merge(env), "git", *CONFIG, "-C", @root, *args, pgroup: true, rlimit_core: 0) do |stdin, stdout, stderr, thread|
|
|
156
|
+
writer = Thread.new do
|
|
157
|
+
stdin.write(input) if input
|
|
158
|
+
rescue IOError, SystemCallError
|
|
159
|
+
nil
|
|
160
|
+
ensure
|
|
161
|
+
stdin.close
|
|
162
|
+
end
|
|
163
|
+
out = Thread.new { stdout.read }
|
|
164
|
+
err = Thread.new { stderr.read }
|
|
165
|
+
timed_out = thread.join(deadline).nil?
|
|
166
|
+
stop(thread) if timed_out
|
|
167
|
+
writer.join
|
|
168
|
+
status = thread.value
|
|
169
|
+
[out.value, timed_out ? "timed out after #{deadline}s" : err.value, status.exitstatus]
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def stop(thread)
|
|
174
|
+
signal(thread.pid, "TERM")
|
|
175
|
+
signal(thread.pid, "KILL") unless thread.join(GRACE)
|
|
176
|
+
thread.join
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def signal(pid, name)
|
|
180
|
+
Process.kill(name, -pid)
|
|
181
|
+
rescue Errno::ESRCH
|
|
182
|
+
nil
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def batch
|
|
186
|
+
@batch ||= begin
|
|
187
|
+
stdin, stdout, thread = Open3.popen2(ENV_DEFAULTS.reject { |name, _| ENV.key?(name) }, "git", *CONFIG, "-C", @root, "cat-file", "--batch", pgroup: true, rlimit_core: 0)
|
|
188
|
+
stdin.timeout = stdout.timeout = @deadline
|
|
189
|
+
[stdin, stdout, thread]
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Riptide
|
|
4
|
+
class Store
|
|
5
|
+
# The rest of a harvesting build's fetch, once every stamp is local:
|
|
6
|
+
# deletes units nobody wrote for EXPIRY, works out which PRs landed in
|
|
7
|
+
# HEAD, and fetches their runs. Every other PR's stamp is dropped
|
|
8
|
+
# locally, along with every other branch's, so their open failures
|
|
9
|
+
# don't count here.
|
|
10
|
+
class Harvest
|
|
11
|
+
def initialize(git, build, remote)
|
|
12
|
+
@git = git
|
|
13
|
+
@build = build
|
|
14
|
+
@remote = remote
|
|
15
|
+
@own = [build.key, build.upstream_key].compact
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def run
|
|
19
|
+
stamps = @git.output("for-each-ref", "--format=%(refname) %(objectname) %(taggerdate:unix)", STAMPS).lines.map(&:split)
|
|
20
|
+
now = Time.now.to_i
|
|
21
|
+
aged, live = stamps.partition { |ref, _, at| !@own.include?(key_of(ref)) && now - at.to_i > EXPIRY }
|
|
22
|
+
age(aged)
|
|
23
|
+
prs = live.select { |ref, _, _| key_of(ref).start_with?("prs/") }
|
|
24
|
+
targeting = prs.select { |_, tag, _| @git.read("#{tag}:landing/target") == @build.branch }
|
|
25
|
+
landed = landed_of(targeting)
|
|
26
|
+
keep = (landed || targeting).map(&:first)
|
|
27
|
+
@git.delete_refs(stamps.map(&:first).reject { |ref| @own.include?(key_of(ref)) || keep.include?(ref) })
|
|
28
|
+
fetch_runs(landed) if landed&.any?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def key_of(stamp) = stamp.delete_prefix(STAMPS).split("/", 2).last
|
|
34
|
+
|
|
35
|
+
def head_entries = @head_entries ||= @git.entries("HEAD")
|
|
36
|
+
|
|
37
|
+
# The PRs whose changes HEAD already has. If the check itself fails,
|
|
38
|
+
# nil: nothing is harvested, and every targeting PR's failures count.
|
|
39
|
+
def landed_of(targeting)
|
|
40
|
+
verdicts = {}
|
|
41
|
+
targeting.select { |_, tag, _| verdicts[@git.resolve("#{tag}:landing")] ||= landed?(tag) }
|
|
42
|
+
rescue StandardError => e
|
|
43
|
+
warn "[riptide] couldn't tell which PRs landed, so none are harvested: #{e.message}"
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Replays the PR's changed paths onto HEAD's, from the merge base. If
|
|
48
|
+
# that's clean and changes nothing, HEAD has the PR's changes however
|
|
49
|
+
# it merged. Only the PR's paths take part, so every other path is
|
|
50
|
+
# the same on all three sides.
|
|
51
|
+
def landed?(tag)
|
|
52
|
+
base = @git.entries(@git.resolve("#{tag}:landing/base"))
|
|
53
|
+
tested = @git.entries(@git.resolve("#{tag}:landing/tested"))
|
|
54
|
+
paths = base.keys | tested.keys
|
|
55
|
+
head = @git.write_tree(head_entries.slice(*paths))
|
|
56
|
+
@git.merge_tree(@git.write_tree(base), head, @git.write_tree(tested)) == head
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Deletes idle units in one atomic push. Each stamp is leased, which
|
|
60
|
+
# covers its runs too, since every write rewrites the stamp. A unit
|
|
61
|
+
# revived meanwhile fails the push, so it's retried once without.
|
|
62
|
+
def age(aged)
|
|
63
|
+
units = aged.map { |stamp, tag, _| [stamp, tag] }
|
|
64
|
+
until units.empty?
|
|
65
|
+
leases = units.map { |stamp, tag| "--force-with-lease=#{stamp}:#{tag}" }
|
|
66
|
+
deletes = units.flat_map { |stamp, _| [":#{stamp}", ":#{stamp.sub(STAMPS, RUNS)}"] }
|
|
67
|
+
out, status = @git.run("push", "--porcelain", "--atomic", *leases, @remote, *deletes, allow: [0, 1])
|
|
68
|
+
break if status.zero?
|
|
69
|
+
|
|
70
|
+
moved = out.lines.grep(/stale info/).map { |line| line.split("\t")[1].to_s.split(":").last }
|
|
71
|
+
break if moved.empty?
|
|
72
|
+
|
|
73
|
+
units.reject! { |stamp, _| moved.include?(stamp) }
|
|
74
|
+
end
|
|
75
|
+
rescue Error => e
|
|
76
|
+
warn "[riptide] couldn't age out idle units: #{e.message}"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def fetch_runs(landed)
|
|
80
|
+
specs = landed.map { |stamp, _, _| runs = stamp.sub(STAMPS, RUNS); "+#{runs}:#{runs}" }
|
|
81
|
+
@git.run("fetch", "--quiet", "--atomic", "--no-tags", "--refmap=", "--no-write-fetch-head", @remote, *specs)
|
|
82
|
+
rescue Error => e
|
|
83
|
+
warn "[riptide] couldn't fetch the runs of PRs that landed, so none are harvested: #{e.message}"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|