riptide 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,458 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "json"
5
+ require "set"
6
+
7
+ module Riptide
8
+ # One unit's passing runs, stored the way selection reads them: from each
9
+ # file version to the runs and shared sets that used each part of it.
10
+ # Every directory is split into 256 shards, so a publish rewrites only the
11
+ # shards whose contents moved.
12
+ #
13
+ # meta the next free id
14
+ # tests/<ab> per test, its file and its kept runs as [id, at]
15
+ # files/<ab> per file version, per path, its postings
16
+ # listings/<ab> per glob, fixture table or absent name, its postings by value
17
+ # members/<ab> per shared set, the runs that point at it
18
+ # scopes per shared set from a test class's or the process's
19
+ # setup, which of the two
20
+ # names/<ab> per constant name, the runs and sets that read it
21
+ # manifest/<ab> per path, its recorded versions
22
+ # loads/<ab> per file version, which classes each line ran for
23
+ # bodies/<ab> per run, its test body's statements
24
+ # blobs/<ab>/<sha> pins that keep recorded versions fetchable
25
+ #
26
+ # A posting list is sorted ids stored as gaps. Runs and sets share one
27
+ # id space, and an id stays the same for as long as the unit keeps it.
28
+ #
29
+ # Failures aren't runs. They're open marks, kept in the unit's stamp and
30
+ # handed in as +marks+: { test => Set of mark ids }.
31
+ class Index
32
+ # Goes up whenever what a run records changes meaning, like how a method
33
+ # is named, so older runs read as none and their tests run once.
34
+ FORMAT = 8
35
+ # The parts of a file a change can name, which selection matches on.
36
+ FACETS = %w[methods blocks owners tables].freeze
37
+ # Plus the lines that ran, which say which way through a method a test
38
+ # went. Selection doesn't need them; the audit does.
39
+ POSTED = [*FACETS, "lines"].freeze
40
+
41
+ Run = Struct.new(:id, :test, :at, keyword_init: true)
42
+
43
+ attr_reader :ref, :marks
44
+
45
+ # +blobs+ returns a tree path's parsed content, or nil. +paths+ lists
46
+ # every path the tree holds.
47
+ def initialize(blobs:, paths:, ref: nil, marks: {})
48
+ @blobs = blobs
49
+ @paths = paths
50
+ @ref = ref
51
+ @marks = marks
52
+ @cache = {}
53
+ @postings = {}
54
+ @members = {}
55
+ @readers = {}
56
+ end
57
+
58
+ # An index over content already in memory, like the logs.
59
+ def self.of(content, marks: {}) = new(blobs: content.method(:[]), paths: content.keys, marks: marks)
60
+
61
+ # An index with no runs, for a unit whose runs can't be read here.
62
+ def self.empty(ref: nil, marks: {}) = new(blobs: ->(_) {}, paths: [], ref: ref, marks: marks)
63
+
64
+ def self.shard(key) = Digest::SHA1.hexdigest(key)[0, 2]
65
+ def self.test_key(test) = JSON.generate(test)
66
+ def self.id_shard(id) = format("%02x", id % 256)
67
+
68
+ def self.gaps(ids)
69
+ previous = 0
70
+ ids.sort.map { |id| (id - previous).tap { previous = id } }
71
+ end
72
+
73
+ def self.ids(gaps)
74
+ sum = 0
75
+ gaps.map { |gap| sum += gap }
76
+ end
77
+
78
+ # { test => [Run, ...] }, newest first.
79
+ def runs
80
+ @runs ||= @paths.grep(%r{\Atests/}).each_with_object({}) do |path, result|
81
+ blob(path).each do |klass, name, _file, test_runs|
82
+ result[[klass, name]] = test_runs.map { |id, at| Run.new(id: id, test: [klass, name], at: at) }
83
+ end
84
+ end
85
+ end
86
+
87
+ def run_ids
88
+ @run_ids ||= runs.values.flatten.to_set(&:id)
89
+ end
90
+
91
+ def manifest
92
+ @manifest ||= @paths.grep(%r{\Amanifest/}).each_with_object({}) { |path, all| all.merge!(blob(path)) }
93
+ end
94
+
95
+ # { "all" => ids, "whole" => ids, facet => { name => ids } } for the
96
+ # holders that recorded +path+ at +version+.
97
+ def postings(path, version)
98
+ @postings[[path, version]] ||= begin
99
+ raw = Env.pseudo?(path) ? blob("listings/#{Index.shard(path)}")&.dig(path, version) : blob("files/#{version[0, 2]}")&.dig(version, path)
100
+ raise Error, "#{@ref || "the logs"} has no postings for #{path} at #{version}" unless raw
101
+
102
+ decode_postings(raw)
103
+ end
104
+ end
105
+
106
+ # "class" or "process" for a set from a test class's or the process's
107
+ # setup, nil for a run or a class the test used.
108
+ def scope(id) = (blob("scopes") || {})[id.to_s]
109
+
110
+ # { run id => how +holders+ reach it }: :test through the run or a class
111
+ # it used, otherwise :class or :process for a setup's set. A run reached
112
+ # more than one way counts the closest.
113
+ def reach(holders)
114
+ holders.each_with_object({}) do |id, result|
115
+ if run_ids.include?(id)
116
+ result[id] = :test
117
+ else
118
+ via = scope(id)&.to_sym || :test
119
+ members(id).each { |run| result[run] = closer(result[run], via) }
120
+ end
121
+ end
122
+ end
123
+
124
+ # Yields each part of +change+ to +path+, recorded here at +version+,
125
+ # with the runs and sets that recorded that part: [:whole, nil, ids] for
126
+ # the ones that depended on the whole file, [facet, name, ids] for a
127
+ # method, block, owner or table, and for a changed constant
128
+ # [:constant, name, ids] for its readers and [:named_constant, name,
129
+ # ids] for those that looked up its last segment by name. A constant can
130
+ # be read anywhere, so its readers can include ones that never recorded
131
+ # +path+.
132
+ def changed_holders(path, version, change)
133
+ postings = postings(path, version)
134
+ yield :whole, nil, postings["whole"] || []
135
+ FACETS.each do |facet|
136
+ by_name = postings[facet] || {}
137
+ change.public_send(facet).each do |name|
138
+ ids = by_name[name.to_s]
139
+ yield facet, name, ids if ids
140
+ end
141
+ end
142
+ change.constants.each do |name|
143
+ yield :constant, name, readers(name)
144
+ yield :named_constant, name, readers("*::#{name.split("::").last}")
145
+ end
146
+ end
147
+
148
+ def members(set_id)
149
+ @members[set_id] ||= Index.ids(blob("members/#{Index.id_shard(set_id)}")&.fetch(set_id.to_s, nil) || [])
150
+ end
151
+
152
+ # The runs and sets that read the constant +name+.
153
+ def readers(name)
154
+ @readers[name] ||= Index.ids(blob("names/#{Index.shard(name)}")&.fetch(name, nil) || [])
155
+ end
156
+
157
+ # The runs among +holders+, plus the runs that point at the sets among
158
+ # them.
159
+ def runs_for(holders)
160
+ holders.each_with_object(Set.new) do |id, result|
161
+ run_ids.include?(id) ? result << id : result.merge(members(id))
162
+ end
163
+ end
164
+
165
+ REACH = %i[test class process].freeze
166
+
167
+ def closer(one, other) = [one, other].compact.min_by { |via| REACH.index(via) }
168
+
169
+ def load_facts(sha) = blob("loads/#{sha[0, 2]}")&.fetch(sha, nil)
170
+
171
+ def pins = @paths.grep(%r{\Ablobs/}).to_set { |path| path.split("/").last }
172
+
173
+ def model = Model.decode(self)
174
+
175
+ # Yields each blob under +prefix+ with its content.
176
+ def each_blob(prefix)
177
+ @paths.each { |path| yield path, blob(path) if path.start_with?(prefix) }
178
+ end
179
+
180
+ def blob(path)
181
+ return @cache[path] if @cache.key?(path)
182
+
183
+ @cache[path] = @blobs.call(path)
184
+ end
185
+
186
+ private
187
+
188
+ def decode_postings(raw)
189
+ raw.to_h do |key, value|
190
+ [key, value.is_a?(Hash) ? value.transform_values { |gaps| Index.ids(gaps) } : Index.ids(value)]
191
+ end
192
+ end
193
+
194
+ # The forward form of an index, for publishing to change: runs and sets
195
+ # by id, each with the files it recorded and the constants it read.
196
+ class Model
197
+ # One top-level statement of a run's test body, in the order they ran:
198
+ # where it starts, the set of what it depended on, how many
199
+ # milliseconds it took, a digest of what each of its assertions or mock
200
+ # expectations compared, whether it wrote data, and the earlier
201
+ # statements, by position, whose writes it read.
202
+ Statement = Struct.new(:line, :set, :ms, :compared, :wrote, :reads, keyword_init: true)
203
+
204
+ # A run's +body+ is its statements, or empty for a test whose body
205
+ # couldn't be split, +ms+ how long the whole test took, +body_line+ the
206
+ # line its method or block starts on, +body_file+ where the body is
207
+ # defined when that isn't the file of the test's class, like a module
208
+ # of shared tests, and +inherited_from+ the test class whose test it is
209
+ # when the test's class only inherits it. Selection never reads them.
210
+ Holder = Struct.new(:test, :at, :sets, :files, :constants, :scope, :body, :ms, :body_line, :body_file, :inherited_from, keyword_init: true) do
211
+ def run? = !test.nil?
212
+
213
+ # What the run or set depended on directly, and where a set came
214
+ # from. Two runs with the same key and the same sets get selected
215
+ # the same way.
216
+ def key = @key ||= Digest::SHA1.hexdigest(JSON.generate([files, constants.sort, scope]))
217
+ end
218
+
219
+ attr_reader :holders, :loads, :pins, :test_files
220
+
221
+ # Log facets carry the constants a file's code read. The model keeps
222
+ # them per holder, since a changed constant matches a read anywhere.
223
+ def self.split(inputs)
224
+ constants = inputs.values.flat_map { |facets| facets["constants"] || [] }
225
+ [inputs.transform_values { |facets| facets.except("constants") }, constants]
226
+ end
227
+
228
+ def initialize(holders: {}, loads: {}, pins: Set.new, next_id: 0, test_files: {})
229
+ @holders = holders
230
+ @loads = loads
231
+ @pins = pins
232
+ @next_id = next_id
233
+ @test_files = test_files
234
+ @set_ids = {}
235
+ @tests = Hash.new { |h, k| h[k] = [] }
236
+ holders.each { |id, holder| holder.run? ? @tests[holder.test] << id : @set_ids[holder.key] = id }
237
+ end
238
+
239
+ def set(id) = @holders.fetch(id)
240
+ def runs = @tests.transform_values { |ids| ids.map { |id| [id, @holders[id]] } }
241
+
242
+ # The id of a set with these files, adding it when it's new. +scope+
243
+ # is "class" or "process" for a setup's set.
244
+ def add_set(files, constants, scope: nil)
245
+ holder = Holder.new(files: canonical(files), constants: constants.to_set, scope: scope)
246
+ @set_ids[holder.key] ||= add(holder)
247
+ end
248
+
249
+ # Adds a passing run unless the test's newest run has the same
250
+ # dependencies, which selects nothing differently. That one keeps the
251
+ # faster timings: a stalled database or a busy machine only ever adds
252
+ # time, so a test's fastest run is what it costs. +file+ is where the
253
+ # test is defined. Returns the run's id, or nil when it was dropped.
254
+ def add_run(test:, at:, sets:, files:, constants:, file: nil, body: [], ms: nil, body_line: nil, body_file: nil, inherited_from: nil)
255
+ @test_files[test] = file if file
256
+ holder = Holder.new(test: test, at: at, sets: sets.uniq.sort, files: canonical(files), constants: constants.to_set, body: body, ms: ms,
257
+ body_line: body_line, body_file: body_file, inherited_from: inherited_from)
258
+ newest = @tests[test].map { |id| @holders[id] }.max_by(&:at)
259
+ if newest && same_dependencies?(newest, holder)
260
+ newest.ms, newest.body = ms, body if ms && (newest.ms.nil? || ms < newest.ms)
261
+ return nil
262
+ end
263
+
264
+ add(holder).tap { |id| @tests[test] << id }
265
+ end
266
+
267
+ def merge_loads(sha, lines)
268
+ known = @loads[sha] ||= {}
269
+ lines.each do |line, fact|
270
+ had = known[line.to_s]
271
+ known[line.to_s] = had ? { "owners" => (had["owners"] | fact["owners"]).sort, "called" => had["called"] || fact["called"] } : fact
272
+ end
273
+ end
274
+
275
+ # Keeps each test's newest +count+ runs, and only the tests in +tests+
276
+ # when given. Then drops the sets and load facts nothing refers to.
277
+ def retain(count, tests: nil)
278
+ keep = tests&.to_set
279
+ @tests.each do |test, ids|
280
+ kept = keep && !keep.include?(test) ? [] : ids.sort_by { |id| -@holders[id].at }.first(count)
281
+ (ids - kept).each { |id| @holders.delete(id) }
282
+ ids.replace(kept)
283
+ end
284
+ @tests.reject! { |_, ids| ids.empty? }
285
+ @test_files.select! { |test, _| @tests.key?(test) }
286
+ used = @tests.values.flatten.flat_map { |id| @holders[id].sets }.to_set
287
+ @holders.select! { |id, holder| holder.run? || used.include?(id) }
288
+ @set_ids.select! { |_, id| @holders.key?(id) }
289
+ versions = self.versions
290
+ @loads.select! { |sha, _| versions.include?(sha) }
291
+ end
292
+
293
+ # Every file version a run or set recorded.
294
+ def versions
295
+ @holders.values.flat_map { |holder| holder.files.filter_map { |path, facets| facets["sha"] unless Env.pseudo?(path) } }.to_set
296
+ end
297
+
298
+ # The tree's content as { path => JSON-ready value }, pins aside.
299
+ def encode
300
+ postings = Hash.new { |h, k| h[k] = {} }
301
+ names = Hash.new { |h, k| h[k] = [] }
302
+ members = Hash.new { |h, k| h[k] = [] }
303
+ manifest = Hash.new { |h, k| h[k] = Set.new }
304
+
305
+ @holders.each do |id, holder|
306
+ holder.files.each do |path, facets|
307
+ manifest[path] << facets["sha"]
308
+ posting = postings[[path, facets["sha"]]]
309
+ (posting["all"] ||= []) << id
310
+ (posting["whole"] ||= []) << id if facets["whole"]
311
+ POSTED.each { |facet| facets[facet]&.each { |name| ((posting[facet] ||= {})[name.to_s] ||= []) << id } }
312
+ end
313
+ holder.constants.each { |name| names[name] << id }
314
+ holder.sets&.each { |set_id| members[set_id] << id }
315
+ end
316
+
317
+ content = {}
318
+ postings.each do |(path, version), posting|
319
+ if Env.pseudo?(path)
320
+ ((content["listings/#{Index.shard(path)}"] ||= {})[path] ||= {})[version] = encode_posting(posting)
321
+ else
322
+ ((content["files/#{version[0, 2]}"] ||= {})[version] ||= {})[path] = encode_posting(posting)
323
+ end
324
+ end
325
+ @tests.each do |test, ids|
326
+ runs = ids.map { |id| [id, @holders[id].at] }.sort_by { |id, at| [-at, id] }
327
+ (content["tests/#{Index.shard(Index.test_key(test))}"] ||= []) << [*test, @test_files[test], runs]
328
+ end
329
+ members.each { |id, runs| (content["members/#{Index.id_shard(id)}"] ||= {})[id.to_s] = Index.gaps(runs) }
330
+ @holders.each do |id, holder|
331
+ next unless holder.run? && holder.body&.any?
332
+
333
+ body = [holder.ms, encode_body(holder.body), holder.body_line, holder.body_file, holder.inherited_from]
334
+ body.pop while body.last.nil?
335
+ (content["bodies/#{Index.id_shard(id)}"] ||= {})[id.to_s] = body
336
+ end
337
+ scopes = @holders.filter_map { |id, holder| [id.to_s, holder.scope] if holder.scope }.sort.to_h
338
+ content["scopes"] = scopes unless scopes.empty?
339
+ names.each { |name, ids| (content["names/#{Index.shard(name)}"] ||= {})[name] = Index.gaps(ids) }
340
+ @loads.each { |sha, lines| (content["loads/#{sha[0, 2]}"] ||= {})[sha] = lines.sort_by { |line, _| line.to_i }.to_h }
341
+ manifest.each { |path, versions| (content["manifest/#{Index.shard(path)}"] ||= {})[path] = versions.sort }
342
+ content.transform_values! { |value| value.is_a?(Array) ? value.sort_by { |entry| entry.first(2) } : deep_sort(value) }
343
+ content["meta"] = { "next" => @next_id }
344
+ content
345
+ end
346
+
347
+ def self.decode(index)
348
+ holders = Hash.new { |h, id| h[id] = Holder.new(files: {}, constants: Set.new) }
349
+ test_files = {}
350
+ index.each_blob("tests/") do |_, entries|
351
+ entries.each do |klass, name, file, test_runs|
352
+ test_files[[klass, name]] = file if file
353
+ test_runs.each do |id, at|
354
+ holder = holders[id]
355
+ holder.test = [klass, name]
356
+ holder.at = at
357
+ holder.sets = []
358
+ end
359
+ end
360
+ end
361
+ index.each_blob("members/") do |_, sets|
362
+ sets.each do |id, runs|
363
+ # A set with no files still belongs to its runs.
364
+ holders[id.to_i]
365
+ Index.ids(runs).each { |run| holders[run].sets << id.to_i }
366
+ end
367
+ end
368
+ index.each_blob("names/") do |_, names|
369
+ names.each { |name, ids| Index.ids(ids).each { |id| holders[id].constants << name } }
370
+ end
371
+ index.each_blob("files/") do |_, by_version|
372
+ by_version.each { |version, by_path| by_path.each { |file, posting| add_postings(holders, file, version, posting) } }
373
+ end
374
+ index.each_blob("listings/") do |_, by_path|
375
+ by_path.each { |file, values| values.each { |value, posting| add_postings(holders, file, value, posting) } }
376
+ end
377
+ (index.blob("scopes") || {}).each { |id, scope| holders[id.to_i].scope = scope }
378
+ index.each_blob("bodies/") do |_, bodies|
379
+ bodies.each do |id, (ms, statements, body_line, body_file, inherited_from)|
380
+ holder = holders[id.to_i]
381
+ holder.ms = ms
382
+ holder.body = decode_body(statements)
383
+ holder.body_line = body_line
384
+ holder.body_file = body_file
385
+ holder.inherited_from = inherited_from
386
+ end
387
+ end
388
+ loads = {}
389
+ index.each_blob("loads/") { |_, by_sha| loads.merge!(by_sha) }
390
+ holders.default_proc = nil
391
+ holders.each_value do |holder|
392
+ holder.body ||= [] if holder.run?
393
+ holder.sets&.sort!
394
+ holder.files = holder.files.sort.to_h { |path, facets| [path, facets.sort.to_h] }
395
+ end
396
+ new(holders: holders, loads: loads, pins: index.pins, next_id: index.blob("meta")&.fetch("next") || 0, test_files: test_files)
397
+ end
398
+
399
+ # Lines and the statements a statement read from go as gaps, from the
400
+ # last line and back from the statement itself.
401
+ def self.decode_body(entries)
402
+ line = 0
403
+ entries.each_with_index.map do |(gap, set, ms, compared, wrote, reads), position|
404
+ line += gap
405
+ Statement.new(line: line, set: set, ms: ms, compared: compared, wrote: wrote == 1, reads: reads.map { |back| position - back })
406
+ end
407
+ end
408
+
409
+ def self.add_postings(holders, path, version, posting)
410
+ all = Index.ids(posting["all"])
411
+ all.each { |id| holders[id].files[path] = { "sha" => version } }
412
+ Index.ids(posting["whole"] || []).each { |id| holders[id].files[path]["whole"] = true }
413
+ POSTED.each do |facet|
414
+ posting[facet]&.each do |name, ids|
415
+ value = %w[blocks lines].include?(facet) ? Integer(name) : name
416
+ Index.ids(ids).each { |id| (holders[id].files[path][facet] ||= []) << value }
417
+ end
418
+ all.each { |id| holders[id].files[path][facet]&.sort! }
419
+ end
420
+ end
421
+
422
+ private
423
+
424
+ def add(holder)
425
+ id = @next_id
426
+ @next_id += 1
427
+ @holders[id] = holder
428
+ id
429
+ end
430
+
431
+ def same_dependencies?(one, other)
432
+ one.key == other.key && one.sets.map { |id| set(id).key }.sort == other.sets.map { |id| set(id).key }.sort
433
+ end
434
+
435
+ # Facet values deduplicated and sorted, with constants held apart.
436
+ def canonical(files)
437
+ files.sort.to_h do |path, facets|
438
+ [path, facets.except("constants").sort.to_h { |facet, value| [facet, value.is_a?(Array) ? value.uniq.sort : value] }]
439
+ end
440
+ end
441
+
442
+ def encode_body(body)
443
+ line = 0
444
+ body.each_with_index.map do |statement, position|
445
+ gap = statement.line - line
446
+ line = statement.line
447
+ [gap, statement.set, statement.ms, statement.compared.sort, statement.wrote ? 1 : 0, statement.reads.map { |earlier| position - earlier }.sort]
448
+ end
449
+ end
450
+
451
+ def encode_posting(posting)
452
+ posting.to_h { |key, value| [key, value.is_a?(Hash) ? value.transform_values { |ids| Index.gaps(ids) } : Index.gaps(value)] }
453
+ end
454
+
455
+ def deep_sort(value) = value.is_a?(Hash) ? value.sort.to_h { |key, inner| [key, deep_sort(inner)] } : value
456
+ end
457
+ end
458
+ end