riptide 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,193 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "open3"
5
+ require "set"
6
+ require "tempfile"
7
+
8
+ module Riptide
9
+ # The git plumbing the store needs, run against one repository. Objects
10
+ # are read through one long-lived `git cat-file --batch`.
11
+ #
12
+ # Every command runs in its own process group under a deadline, so a
13
+ # stalled call can't hold up a build: the group gets TERM, which makes
14
+ # git drop its locks, then KILL. The batch reader's deadline is per
15
+ # object. Nothing prompts, nothing starts a gc while tests read .git, and
16
+ # an aborting git leaves no core file.
17
+ class Git
18
+ DEADLINE = 120
19
+ GRACE = 5
20
+ CONFIG = %w[-c gc.auto=0 -c maintenance.auto=false].freeze
21
+ ENV_DEFAULTS = { "GIT_TERMINAL_PROMPT" => "0", "GIT_SSH_COMMAND" => "ssh -o BatchMode=yes -o ConnectTimeout=15",
22
+ "GIT_HTTP_LOW_SPEED_LIMIT" => "1000", "GIT_HTTP_LOW_SPEED_TIME" => "30" }.freeze
23
+
24
+ def initialize(root, deadline: DEADLINE)
25
+ @root = root
26
+ @deadline = deadline
27
+ end
28
+
29
+ # [stdout, exit status], where a signal exit is nil. Raises unless the
30
+ # status is in +allow+.
31
+ def run(*args, input: nil, env: {}, allow: [0], deadline: @deadline)
32
+ out, err, status = spawn(args, input, env, deadline)
33
+ raise Error, "git #{args.first} failed: #{err.strip}" unless allow.include?(status)
34
+
35
+ [out, status]
36
+ end
37
+
38
+ def output(*args, **options) = run(*args, **options).first
39
+
40
+ # A blob's content, or nil when the repository doesn't have it.
41
+ def read(sha)
42
+ batch_in, batch_out, = batch
43
+ batch_in.puts(sha)
44
+ header = batch_out.gets
45
+ raise IOError, "the reader exited" if header.nil?
46
+ return nil if header.end_with?("missing\n")
47
+
48
+ size = header.split.last.to_i
49
+ content = batch_out.read(size)
50
+ batch_out.read(1)
51
+ content
52
+ rescue IO::TimeoutError, IOError, SystemCallError => e
53
+ # A stalled or dead reader starts over on the next read.
54
+ if @batch
55
+ stdin, stdout, thread = @batch
56
+ @batch = nil
57
+ [stdin, stdout].each { |io| io.close unless io.closed? }
58
+ stop(thread)
59
+ end
60
+ raise Error, e.is_a?(IO::TimeoutError) ? "git cat-file timed out after #{@deadline}s" : "git cat-file failed: #{e.message}"
61
+ end
62
+
63
+ # Every path under the root git tracks or would track, relative to the
64
+ # root: committed, staged, or untracked and not ignored.
65
+ def files
66
+ output("ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0").to_set
67
+ end
68
+
69
+ # Writes each of +blobs+ ({ path => content }) in one fast-import and
70
+ # returns { path => blob sha }. Ones already in +known+ aren't written
71
+ # again.
72
+ def write_blobs(blobs, known: Set.new)
73
+ shas = blobs.transform_values { |content| Digest::SHA1.hexdigest("blob #{content.bytesize}\0#{content}") }
74
+ fresh = blobs.reject { |path, _| known.include?(shas[path]) }.values.uniq
75
+ unless fresh.empty?
76
+ stream = fresh.map { |content| "blob\ndata #{content.bytesize}\n#{content}\n" }.join << "done\n"
77
+ run("fast-import", "--quiet", "--done", input: stream)
78
+ end
79
+ shas
80
+ end
81
+
82
+ # { ref => object } for this repository's refs under +prefix+.
83
+ def refs(prefix)
84
+ output("for-each-ref", "--format=%(refname) %(objectname)", prefix).lines.to_h { |line| line.split.first(2) }
85
+ end
86
+
87
+ # Deletes +names+ in one transaction.
88
+ def delete_refs(names)
89
+ run("update-ref", "--stdin", input: names.map { |name| "delete #{name}\n" }.join) unless names.empty?
90
+ end
91
+
92
+ def merge_base(one, other)
93
+ out, status = run("merge-base", one, other, allow: [0, 1, 128])
94
+ status.zero? ? out.strip : nil
95
+ end
96
+
97
+ # The tree from merging +theirs+ into +ours+ as if +base+ were their
98
+ # common ancestor, or nil when it conflicts. All three can be trees.
99
+ def merge_tree(base, ours, theirs)
100
+ out, status = run("merge-tree", "--write-tree", "--no-messages", "--merge-base=#{base}", ours, theirs, allow: [0, 1])
101
+ status.zero? ? out.lines.first.strip : nil
102
+ end
103
+
104
+ # The object +rev+ names, or nil.
105
+ def resolve(rev)
106
+ out, status = run("rev-parse", "--verify", "--quiet", rev, allow: [0, 1, 128])
107
+ status.zero? ? out.strip : nil
108
+ end
109
+
110
+ # { path => [mode, object] } for every file in +tree+, recursively, with
111
+ # paths from the top of the tree. Without --full-tree, git would list
112
+ # only what's under the root when the root is a subdirectory.
113
+ def entries(tree)
114
+ return {} if tree.nil?
115
+
116
+ output("ls-tree", "-r", "-z", "--full-tree", tree).split("\0").to_h do |entry|
117
+ info, path = entry.split("\t", 2)
118
+ mode, _, object = info.split
119
+ [path, [mode, object]]
120
+ end
121
+ end
122
+
123
+ # { path => blob sha } for every file in a tree.
124
+ def tree(object) = entries(object).transform_values(&:last)
125
+
126
+ # The root's tree in the checked-out commit, whose paths are the ones
127
+ # riptide records: relative to the root, which can be a subdirectory of
128
+ # the repository.
129
+ def head_tree = "HEAD:#{@prefix ||= output("rev-parse", "--show-prefix").strip}"
130
+
131
+ # A tree holding +files+ ({ path => sha } or { path => [mode, sha] }),
132
+ # built in a throwaway index so the repository's own is never touched.
133
+ def write_tree(files)
134
+ Tempfile.create("riptide-index") do |index|
135
+ File.delete(index.path)
136
+ env = { "GIT_INDEX_FILE" => index.path }
137
+ listing = files.sort.map do |path, entry|
138
+ mode, sha = entry.is_a?(Array) ? entry : ["100644", entry]
139
+ "#{mode} #{sha}\t#{path}\0"
140
+ end.join
141
+ run("update-index", "-z", "--add", "--index-info", input: listing, env: env) unless listing.empty?
142
+ output("write-tree", "--missing-ok", env: env).strip
143
+ end
144
+ end
145
+
146
+ # An annotated tag wrapping +tree+, dated now by riptide's own clock.
147
+ def tag_tree(tree, name)
148
+ content = "object #{tree}\ntype tree\ntag #{name}\ntagger riptide <riptide@users.noreply.github.com> #{Time.now.to_i} +0000\n\n#{name}\n"
149
+ output("mktag", input: content).strip
150
+ end
151
+
152
+ private
153
+
154
+ def spawn(args, input, env, deadline)
155
+ Open3.popen3(ENV_DEFAULTS.reject { |name, _| ENV.key?(name) }.merge(env), "git", *CONFIG, "-C", @root, *args, pgroup: true, rlimit_core: 0) do |stdin, stdout, stderr, thread|
156
+ writer = Thread.new do
157
+ stdin.write(input) if input
158
+ rescue IOError, SystemCallError
159
+ nil
160
+ ensure
161
+ stdin.close
162
+ end
163
+ out = Thread.new { stdout.read }
164
+ err = Thread.new { stderr.read }
165
+ timed_out = thread.join(deadline).nil?
166
+ stop(thread) if timed_out
167
+ writer.join
168
+ status = thread.value
169
+ [out.value, timed_out ? "timed out after #{deadline}s" : err.value, status.exitstatus]
170
+ end
171
+ end
172
+
173
+ def stop(thread)
174
+ signal(thread.pid, "TERM")
175
+ signal(thread.pid, "KILL") unless thread.join(GRACE)
176
+ thread.join
177
+ end
178
+
179
+ def signal(pid, name)
180
+ Process.kill(name, -pid)
181
+ rescue Errno::ESRCH
182
+ nil
183
+ end
184
+
185
+ def batch
186
+ @batch ||= begin
187
+ stdin, stdout, thread = Open3.popen2(ENV_DEFAULTS.reject { |name, _| ENV.key?(name) }, "git", *CONFIG, "-C", @root, "cat-file", "--batch", pgroup: true, rlimit_core: 0)
188
+ stdin.timeout = stdout.timeout = @deadline
189
+ [stdin, stdout, thread]
190
+ end
191
+ end
192
+ end
193
+ end
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Riptide
4
+ class Store
5
+ # The rest of a harvesting build's fetch, once every stamp is local:
6
+ # deletes units nobody wrote for EXPIRY, works out which PRs landed in
7
+ # HEAD, and fetches their runs. Every other PR's stamp is dropped
8
+ # locally, along with every other branch's, so their open failures
9
+ # don't count here.
10
+ class Harvest
11
+ def initialize(git, build, remote)
12
+ @git = git
13
+ @build = build
14
+ @remote = remote
15
+ @own = [build.key, build.upstream_key].compact
16
+ end
17
+
18
+ def run
19
+ stamps = @git.output("for-each-ref", "--format=%(refname) %(objectname) %(taggerdate:unix)", STAMPS).lines.map(&:split)
20
+ now = Time.now.to_i
21
+ aged, live = stamps.partition { |ref, _, at| !@own.include?(key_of(ref)) && now - at.to_i > EXPIRY }
22
+ age(aged)
23
+ prs = live.select { |ref, _, _| key_of(ref).start_with?("prs/") }
24
+ targeting = prs.select { |_, tag, _| @git.read("#{tag}:landing/target") == @build.branch }
25
+ landed = landed_of(targeting)
26
+ keep = (landed || targeting).map(&:first)
27
+ @git.delete_refs(stamps.map(&:first).reject { |ref| @own.include?(key_of(ref)) || keep.include?(ref) })
28
+ fetch_runs(landed) if landed&.any?
29
+ end
30
+
31
+ private
32
+
33
+ def key_of(stamp) = stamp.delete_prefix(STAMPS).split("/", 2).last
34
+
35
+ def head_entries = @head_entries ||= @git.entries("HEAD")
36
+
37
+ # The PRs whose changes HEAD already has. If the check itself fails,
38
+ # nil: nothing is harvested, and every targeting PR's failures count.
39
+ def landed_of(targeting)
40
+ verdicts = {}
41
+ targeting.select { |_, tag, _| verdicts[@git.resolve("#{tag}:landing")] ||= landed?(tag) }
42
+ rescue StandardError => e
43
+ warn "[riptide] couldn't tell which PRs landed, so none are harvested: #{e.message}"
44
+ nil
45
+ end
46
+
47
+ # Replays the PR's changed paths onto HEAD's, from the merge base. If
48
+ # that's clean and changes nothing, HEAD has the PR's changes however
49
+ # it merged. Only the PR's paths take part, so every other path is
50
+ # the same on all three sides.
51
+ def landed?(tag)
52
+ base = @git.entries(@git.resolve("#{tag}:landing/base"))
53
+ tested = @git.entries(@git.resolve("#{tag}:landing/tested"))
54
+ paths = base.keys | tested.keys
55
+ head = @git.write_tree(head_entries.slice(*paths))
56
+ @git.merge_tree(@git.write_tree(base), head, @git.write_tree(tested)) == head
57
+ end
58
+
59
+ # Deletes idle units in one atomic push. Each stamp is leased, which
60
+ # covers its runs too, since every write rewrites the stamp. A unit
61
+ # revived meanwhile fails the push, so it's retried once without.
62
+ def age(aged)
63
+ units = aged.map { |stamp, tag, _| [stamp, tag] }
64
+ until units.empty?
65
+ leases = units.map { |stamp, tag| "--force-with-lease=#{stamp}:#{tag}" }
66
+ deletes = units.flat_map { |stamp, _| [":#{stamp}", ":#{stamp.sub(STAMPS, RUNS)}"] }
67
+ out, status = @git.run("push", "--porcelain", "--atomic", *leases, @remote, *deletes, allow: [0, 1])
68
+ break if status.zero?
69
+
70
+ moved = out.lines.grep(/stale info/).map { |line| line.split("\t")[1].to_s.split(":").last }
71
+ break if moved.empty?
72
+
73
+ units.reject! { |stamp, _| moved.include?(stamp) }
74
+ end
75
+ rescue Error => e
76
+ warn "[riptide] couldn't age out idle units: #{e.message}"
77
+ end
78
+
79
+ def fetch_runs(landed)
80
+ specs = landed.map { |stamp, _, _| runs = stamp.sub(STAMPS, RUNS); "+#{runs}:#{runs}" }
81
+ @git.run("fetch", "--quiet", "--atomic", "--no-tags", "--refmap=", "--no-write-fetch-head", @remote, *specs)
82
+ rescue Error => e
83
+ warn "[riptide] couldn't fetch the runs of PRs that landed, so none are harvested: #{e.message}"
84
+ end
85
+ end
86
+ end
87
+ end