rigortype 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -3
  3. data/docs/handbook/08-understanding-errors.md +3 -2
  4. data/docs/install.md +87 -15
  5. data/docs/manual/02-cli-reference.md +2 -2
  6. data/docs/manual/03-configuration.md +5 -0
  7. data/docs/manual/08-skills.md +4 -3
  8. data/docs/manual/12-caching.md +43 -4
  9. data/docs/manual/14-rails-quickstart.md +19 -11
  10. data/lib/rigor/analysis/check_rules/source_arity.rb +163 -59
  11. data/lib/rigor/analysis/check_rules.rb +118 -87
  12. data/lib/rigor/analysis/dependency_recorder.rb +200 -18
  13. data/lib/rigor/analysis/incremental_run_slot/write_guard.rb +266 -0
  14. data/lib/rigor/analysis/incremental_run_slot.rb +171 -0
  15. data/lib/rigor/analysis/incremental_session.rb +273 -49
  16. data/lib/rigor/analysis/plugin_fact_fingerprint.rb +24 -13
  17. data/lib/rigor/analysis/reachability/scan_cache.rb +2 -1
  18. data/lib/rigor/analysis/run_cache_key.rb +27 -8
  19. data/lib/rigor/analysis/run_stats.rb +2 -2
  20. data/lib/rigor/analysis/runner/diagnostic_aggregator.rb +1 -1
  21. data/lib/rigor/analysis/runner/pool_coordinator.rb +2 -2
  22. data/lib/rigor/analysis/runner/project_pre_passes.rb +21 -13
  23. data/lib/rigor/analysis/runner.rb +337 -46
  24. data/lib/rigor/analysis/shadow_harness.rb +27 -0
  25. data/lib/rigor/analysis/source_rbs_gate.rb +157 -0
  26. data/lib/rigor/analysis/template_unit_paths.rb +6 -4
  27. data/lib/rigor/analysis/template_units.rb +15 -5
  28. data/lib/rigor/analysis/worker_session.rb +4 -3
  29. data/lib/rigor/cache/descriptor.rb +40 -1
  30. data/lib/rigor/cache/file_digest.rb +28 -0
  31. data/lib/rigor/cache/incremental_snapshot.rb +70 -8
  32. data/lib/rigor/cache/store.rb +63 -3
  33. data/lib/rigor/cache/value_digest.rb +325 -0
  34. data/lib/rigor/cli/check_command.rb +83 -13
  35. data/lib/rigor/cli/doctor_command.rb +1 -1
  36. data/lib/rigor/cli/effects_report.rb +2 -2
  37. data/lib/rigor/cli/unused_command.rb +3 -3
  38. data/lib/rigor/cli.rb +1 -1
  39. data/lib/rigor/config_audit.rb +3 -2
  40. data/lib/rigor/config_path.rb +29 -0
  41. data/lib/rigor/configuration/severity_profile.rb +5 -0
  42. data/lib/rigor/configuration.rb +6 -5
  43. data/lib/rigor/effects/plugin_facts.rb +6 -6
  44. data/lib/rigor/environment/bundle_sig_discovery.rb +5 -3
  45. data/lib/rigor/environment/lockfile_resolver.rb +10 -1
  46. data/lib/rigor/environment/rbs_collection_discovery.rb +10 -1
  47. data/lib/rigor/environment/rbs_loader.rb +1 -1
  48. data/lib/rigor/environment/source_rbs_synthesis.rb +147 -0
  49. data/lib/rigor/environment.rb +13 -70
  50. data/lib/rigor/inference/block_parameter_binder.rb +29 -13
  51. data/lib/rigor/inference/captured_locals.rb +7 -1
  52. data/lib/rigor/inference/declaration_walk/context.rb +195 -0
  53. data/lib/rigor/inference/declaration_walk/errors.rb +18 -0
  54. data/lib/rigor/inference/declaration_walk/shadow.rb +147 -0
  55. data/lib/rigor/inference/declaration_walk/traversal.rb +441 -0
  56. data/lib/rigor/inference/declaration_walk.rb +7 -0
  57. data/lib/rigor/inference/def_node_resolver.rb +3 -2
  58. data/lib/rigor/inference/definer_resolution.rb +266 -0
  59. data/lib/rigor/inference/expression_typer.rb +136 -45
  60. data/lib/rigor/inference/external_ancestor_resolution.rb +13 -16
  61. data/lib/rigor/inference/guard_rebinding.rb +53 -12
  62. data/lib/rigor/inference/macro_block_self_type.rb +14 -2
  63. data/lib/rigor/inference/method_dispatcher/rbs_dispatch.rb +46 -26
  64. data/lib/rigor/inference/module_function_state.rb +213 -0
  65. data/lib/rigor/inference/multi_target_binder.rb +21 -16
  66. data/lib/rigor/inference/narrowing.rb +147 -17
  67. data/lib/rigor/inference/parameter_inference_collector.rb +10 -9
  68. data/lib/rigor/inference/scope_indexer/certainty.rb +176 -0
  69. data/lib/rigor/inference/scope_indexer/class_cvars_collector.rb +35 -0
  70. data/lib/rigor/inference/scope_indexer/def_nestings_collector.rb +47 -0
  71. data/lib/rigor/inference/scope_indexer/layered_def_nestings.rb +36 -0
  72. data/lib/rigor/inference/scope_indexer/member_layouts_collector.rb +61 -0
  73. data/lib/rigor/inference/scope_indexer/mixin_accumulator.rb +130 -0
  74. data/lib/rigor/inference/scope_indexer/superclasses_collector.rb +64 -0
  75. data/lib/rigor/inference/scope_indexer.rb +1443 -295
  76. data/lib/rigor/inference/singleton_hook_decline.rb +253 -0
  77. data/lib/rigor/inference/statement_evaluator.rb +80 -14
  78. data/lib/rigor/plugin/base.rb +28 -8
  79. data/lib/rigor/plugin/io_boundary.rb +82 -4
  80. data/lib/rigor/plugin/isolation.rb +1 -1
  81. data/lib/rigor/plugin/trust_policy.rb +3 -3
  82. data/lib/rigor/protection/discovery_seed.rb +18 -9
  83. data/lib/rigor/protection/mutation_cache.rb +3 -3
  84. data/lib/rigor/reflection/constant_ancestors.rb +96 -47
  85. data/lib/rigor/reflection/constant_path.rb +22 -7
  86. data/lib/rigor/reflection.rb +2 -3
  87. data/lib/rigor/scope/discovery_index.rb +218 -2
  88. data/lib/rigor/scope/resolution_chain/relevance.rb +165 -0
  89. data/lib/rigor/scope/resolution_chain.rb +884 -0
  90. data/lib/rigor/scope.rb +295 -322
  91. data/lib/rigor/sig_gen/alias_index.rb +1 -1
  92. data/lib/rigor/sig_gen/effect_annotation.rb +2 -2
  93. data/lib/rigor/sig_gen/generator.rb +22 -16
  94. data/lib/rigor/type/combinator.rb +25 -1
  95. data/lib/rigor/version.rb +1 -1
  96. data/plugins/rigor-ffi/lib/rigor/plugin/ffi.rb +10 -0
  97. data/sig/rigor/environment.rbs +2 -0
  98. data/sig/rigor/plugin/io_boundary.rbs +1 -0
  99. data/sig/rigor/scope.rbs +43 -1
  100. data/skills/rigor-project-init/SKILL.md +37 -11
  101. data/skills/rigor-project-init/references/02-configure.md +7 -2
  102. metadata +25 -1
@@ -22,12 +22,52 @@ module Rigor
22
22
  # It is a STACK because captures nest (a memoised callee body evaluating another memo-miss callee): an
23
23
  # outer capture must also see the inner window's reads, or its read-set would be non-transitive and a
24
24
  # later replay would drop the nested callee's edges.
25
+ #
26
+ # Every event reaches every capture open at the time, so an inner capture's sets are always a subset of
27
+ # each outer one's: the inner capture was pushed after the outer, and everything added since reached both.
28
+ # {withhold} swaps in a fresh stack and restores this one untouched, so it keeps the property too.
29
+ # {tee_read} and {tee_missing} rely on it to stop at the first capture that already holds an event.
25
30
  CAPTURE_KEY = :__rigor_dependency_capture_stack__
26
31
  private_constant :CAPTURE_KEY
27
32
 
33
+ # The read sets an {Accumulator} or {Capture} already holds in full, by identity: replayed into it, or
34
+ # captured by a window it saw every event of ({capture}). Both only ever grow and a {ReadSet} is frozen,
35
+ # so replaying such a set into the same target again adds nothing. Recording replays the same stored
36
+ # read set many times over: on Rigor's own `lib`, 71% of the replays into a consumer repeated one
37
+ # already made into it, carrying 65% of the replayed events.
38
+ module ReplayMemory
39
+ def holds?(read_set)
40
+ !@held.nil? && @held.key?(read_set)
41
+ end
42
+
43
+ # Called only once every event of `read_set` is in, so a replay that raises part-way leaves the
44
+ # target unmarked and the next replay adds the rest.
45
+ def hold!(read_set)
46
+ (@held ||= {}.compare_by_identity)[read_set] = true
47
+ end
48
+ end
49
+
50
+ # #1590 — the resolution-chain filings ({file_chain_once}) an {Accumulator} or {Capture} already holds in
51
+ # full: per chain (by identity; the table holds the chain, so its identity cannot be reused while the target
52
+ # lives), the starts filed from it and the discovery index each was filed against. A filing is a pure
53
+ # function of those three, so filing it again into the same target adds nothing.
54
+ module ChainMemory
55
+ def chain_filed?(chain, start, discovery)
56
+ table = @chains&.[](chain)
57
+ !table.nil? && table.key?(start) && table[start].equal?(discovery)
58
+ end
59
+
60
+ def chain_filed!(chain, start, discovery)
61
+ ((@chains ||= {}.compare_by_identity)[chain] ||= {})[start] = discovery
62
+ end
63
+ end
64
+
28
65
  # Mutable per-consumer accumulator. Frozen into a {Record} snapshot when `record_for` returns.
29
66
  class Accumulator
30
- attr_reader :consumer, :sources, :missing, :symbol_sources, :ancestry_sources
67
+ include ReplayMemory
68
+ include ChainMemory
69
+
70
+ attr_reader :consumer, :sources, :missing, :symbol_sources, :ancestry_sources, :suspensions
31
71
 
32
72
  def initialize(consumer)
33
73
  @consumer = consumer
@@ -39,6 +79,15 @@ module Rigor
39
79
  # file-granularity by nature (a superclass edge touches the whole class).
40
80
  @symbol_sources = Hash.new { |h, k| h[k] = Set.new }
41
81
  @ancestry_sources = Set.new
82
+ @held = nil
83
+ @chains = nil
84
+ @suspensions = 0
85
+ end
86
+
87
+ # Counts the {record_for} windows of another consumer opened while this one was recording, which is
88
+ # when a capture window stops seeing only this consumer's events.
89
+ def suspend!
90
+ @suspensions += 1
42
91
  end
43
92
 
44
93
  def snapshot
@@ -65,16 +114,28 @@ module Rigor
65
114
  # self-read filter is applied at {replay} time, mirroring {read_site}.
66
115
  ReadSet = Data.define(:reads, :missing)
67
116
 
117
+ # What every window that saw nothing closes into: most windows {RbsDispatch}'s recording-mode memos
118
+ # open see nothing, and they need neither a copy of two empty Sets nor a place in a target's
119
+ # {ReplayMemory}.
120
+ EMPTY_READ_SET = Ractor.make_shareable(ReadSet.new(reads: Set.new, missing: Set.new))
121
+
68
122
  # Mutable capture accumulator; snapshot into a frozen {ReadSet} when the capture window closes.
69
123
  class Capture
124
+ include ReplayMemory
125
+ include ChainMemory
126
+
70
127
  attr_reader :reads, :missing
71
128
 
72
129
  def initialize
73
130
  @reads = Set.new
74
131
  @missing = Set.new
132
+ @held = nil
133
+ @chains = nil
75
134
  end
76
135
 
77
136
  def snapshot
137
+ return EMPTY_READ_SET if reads.empty? && missing.empty?
138
+
78
139
  ReadSet.new(reads: reads.dup.freeze, missing: missing.dup.freeze)
79
140
  end
80
141
  end
@@ -94,6 +155,7 @@ module Rigor
94
155
  # the previous recorder on exit.
95
156
  def record_for(consumer)
96
157
  previous = Thread.current[KEY]
158
+ previous&.suspend!
97
159
  accumulator = Accumulator.new(consumer.to_s)
98
160
  Thread.current[KEY] = accumulator
99
161
  @mutex.synchronize { @active_count += 1 }
@@ -109,6 +171,13 @@ module Rigor
109
171
  @active_count.positive?
110
172
  end
111
173
 
174
+ # Whether THIS thread's reads reach an accumulator: inside {record_for}, including a {withhold}
175
+ # window. {active?} answers for the process and is the cheap gate; this is the exact one, for a caller
176
+ # that must know whether an empty {capture} means "nothing to record" or "nothing was watching".
177
+ def recording?
178
+ !Thread.current[KEY].nil?
179
+ end
180
+
112
181
  # ADR-84 WD2 — runs the block under an observe-and-forward capture window and returns
113
182
  # `[block_result, read_set]`. Reads recorded during the window reach the current consumer's accumulator
114
183
  # unchanged (forwarding) and are additionally collected into the returned frozen {ReadSet} (observing),
@@ -116,6 +185,8 @@ module Rigor
116
185
  # {active?}; opening a capture with no accumulator active would collect reads no run is recording.
117
186
  def capture
118
187
  captures = (Thread.current[CAPTURE_KEY] ||= [])
188
+ accumulator = Thread.current[KEY]
189
+ suspensions = accumulator&.suspensions
119
190
  capture = Capture.new
120
191
  captures.push(capture)
121
192
  begin
@@ -123,9 +194,28 @@ module Rigor
123
194
  ensure
124
195
  captures.pop
125
196
  end
126
- [result, capture.snapshot]
197
+ read_set = capture.snapshot
198
+ note_captured(read_set, captures, accumulator, suspensions)
199
+ [result, read_set]
127
200
  end
128
201
 
202
+ # Every event a window saw also reached the window outside it, and was filed for the consumer that was
203
+ # recording at the time, except the consumer's own self-reads, which a replay into that consumer drops
204
+ # anyway. So the enclosing window holds the closed window's read set in full, and so does the consumer's
205
+ # accumulator when no other consumer recorded in between. Noting that lets a later replay of the set
206
+ # there skip a pass that could add nothing: on Rigor's own `lib`, two thirds of the first replays into a
207
+ # consumer were of a set captured while that consumer was recording.
208
+ def note_captured(read_set, captures, accumulator, suspensions)
209
+ return if read_set.equal?(EMPTY_READ_SET)
210
+
211
+ captures.last&.hold!(read_set)
212
+ return unless accumulator && Thread.current[KEY].equal?(accumulator)
213
+ return unless accumulator.suspensions == suspensions
214
+
215
+ accumulator.hold!(read_set)
216
+ end
217
+ private_class_method :note_captured
218
+
129
219
  # Issue #992 — runs the block with its reads DETACHED from the current consumer (and from any enclosing
130
220
  # capture) and returns `[block_result, read_set]`, so a caller that learns only afterwards whether its
131
221
  # answer depended on those reads can {replay} them or drop them. `call.wrong-arity` reads a class's whole
@@ -154,21 +244,74 @@ module Rigor
154
244
  # capture so an enclosing capture window stays transitive when a memo hit substitutes for a body walk.
155
245
  def replay(read_set)
156
246
  accumulator = Thread.current[KEY]
157
- return if accumulator.nil? || read_set.nil?
247
+ return if accumulator.nil? || read_set.nil? || read_set.equal?(EMPTY_READ_SET)
158
248
 
159
249
  captures = Thread.current[CAPTURE_KEY]
250
+ tee_read_set(captures, read_set) if captures && !captures.empty?
251
+ # Each set keeps its own insertion order whichever target is filled first, and a skipped replay is one
252
+ # that could only have re-added what the target already holds.
253
+ return if accumulator.holds?(read_set)
254
+
255
+ consumer = accumulator.consumer
160
256
  read_set.reads.each do |pair|
161
- captures&.each { |c| c.reads.add(pair) }
162
257
  path, symbol = pair
163
- next if path == accumulator.consumer
164
-
165
- accumulate_read(accumulator, path, symbol)
258
+ accumulate_read(accumulator, path, symbol) unless path == consumer
166
259
  end
167
- read_set.missing.each do |entry|
168
- captures&.each { |c| c.missing.add(entry) }
169
- accumulator.missing << entry
260
+ missing = accumulator.missing
261
+ read_set.missing.each { |entry| missing << entry }
262
+ accumulator.hold!(read_set)
263
+ end
264
+
265
+ # #1590 — runs the block, which files a resolution chain's edges from entry `start` (nil: the root and the
266
+ # whole chain) against `discovery`, unless the current consumer's accumulator AND the innermost open capture
267
+ # already hold that filing; the block files the same edges every time, so skipping it there changes nothing.
268
+ # Every event reaches every capture open at the time (see CAPTURE_KEY), so the innermost capture holding the
269
+ # filing means every open capture does. A filing the accumulator holds but a capture opened since does not is
270
+ # filed again, so that capture's read set, replayed elsewhere, still carries it; and {withhold} starts a fresh
271
+ # accumulator and capture, which hold nothing. The marks are made only once the block has run, as {hold!}'s are.
272
+ def file_chain_once(chain, start, discovery)
273
+ accumulator = Thread.current[KEY]
274
+ return if accumulator.nil?
275
+
276
+ captures = Thread.current[CAPTURE_KEY]
277
+ open = captures && !captures.empty?
278
+ if accumulator.chain_filed?(chain, start, discovery) &&
279
+ (!open || captures.last.chain_filed?(chain, start, discovery))
280
+ return
170
281
  end
282
+
283
+ yield
284
+ accumulator.chain_filed!(chain, start, discovery)
285
+ captures.each { |capture| capture.chain_filed!(chain, start, discovery) } if open
286
+ end
287
+
288
+ # {replay}'s share for the open captures. The innermost capture having seen `read_set` already means every
289
+ # open capture holds all of it (see CAPTURE_KEY).
290
+ def tee_read_set(captures, read_set)
291
+ innermost = captures.last
292
+ return if innermost.holds?(read_set)
293
+
294
+ read_set.reads.each { |pair| tee_read(captures, pair) }
295
+ read_set.missing.each { |entry| tee_missing(captures, entry) }
296
+ innermost.hold!(read_set)
171
297
  end
298
+ private_class_method :tee_read_set
299
+
300
+ # Adds a read pair to the open captures, innermost first, and stops at the first capture that already holds
301
+ # it: every capture outside that one holds it too (see CAPTURE_KEY). The capture sets end up exactly as
302
+ # adding it to every capture would leave them, in the same insertion order.
303
+ def tee_read(captures, pair)
304
+ index = captures.size - 1
305
+ index -= 1 while index >= 0 && captures[index].reads.add?(pair)
306
+ end
307
+ private_class_method :tee_read
308
+
309
+ # {tee_read} for a name-keyed `missing` entry.
310
+ def tee_missing(captures, entry)
311
+ index = captures.size - 1
312
+ index -= 1 while index >= 0 && captures[index].missing.add?(entry)
313
+ end
314
+ private_class_method :tee_missing
172
315
 
173
316
  # Records that the current consumer read a declaration / body whose definition site is `path_line` (a
174
317
  # `"path:line"` String, or nil). When `symbol` is given (a `"ClassName#method"` String), the read is a
@@ -181,16 +324,26 @@ module Rigor
181
324
  accumulator = Thread.current[KEY]
182
325
  return if accumulator.nil? || path_line.nil?
183
326
 
184
- path = path_line.split(":", 2).first
327
+ path = site_path(path_line)
185
328
  return unless path
186
329
 
187
330
  captures = Thread.current[CAPTURE_KEY]
188
- captures&.each { |c| c.reads.add([path, symbol].freeze) }
331
+ tee_read(captures, [path, symbol].freeze) if captures && !captures.empty?
189
332
  return if path == accumulator.consumer
190
333
 
191
334
  accumulate_read(accumulator, path, symbol)
192
335
  end
193
336
 
337
+ # The path part of a `"path:line"` site: what `path_line.split(":", 2).first` answers, without the Array
338
+ # and the discarded line String.
339
+ def site_path(path_line)
340
+ colon = path_line.byteindex(":")
341
+ return path_line.byteslice(0, colon) if colon
342
+
343
+ path_line.empty? ? nil : path_line.dup
344
+ end
345
+ private_class_method :site_path
346
+
194
347
  # Records a NAME-KEYED cross-file dependency: this consumer's answer depends on what the project makes
195
348
  # of `name` under `kind`, so a recheck must re-analyze it when that changes. The recorded key is
196
349
  # inverted into `negative_dependents` and matched against the producer for its kind.
@@ -206,10 +359,38 @@ module Rigor
206
359
  return if accumulator.nil?
207
360
 
208
361
  entry = "#{kind}:#{name}"
209
- Thread.current[CAPTURE_KEY]&.each { |c| c.missing.add(entry) }
362
+ captures = Thread.current[CAPTURE_KEY]
363
+ tee_missing(captures, entry) if captures && !captures.empty?
210
364
  accumulator.missing.add(entry)
211
365
  end
212
366
 
367
+ # {read_name} keyed on the LAST SEGMENT of a constant path, `name.delete_prefix("::").split("::").last`,
368
+ # which is the key the `class:` and `constant:` name edges use. The segment is found by scanning for the
369
+ # last separator rather than by building every segment: on Rigor's own `lib` that split cost about seven
370
+ # objects a call over half a million calls. A name the scan could read differently from `split` (one
371
+ # that is not ASCII, or carries a run of colons or a trailing one) takes `split`'s answer.
372
+ def read_last_segment(kind, name)
373
+ return if Thread.current[KEY].nil?
374
+
375
+ segment = last_segment(name)
376
+ read_name(kind, segment) if segment
377
+ end
378
+
379
+ def last_segment(name)
380
+ return name.delete_prefix("::").split("::").last unless plain_constant_path?(name)
381
+
382
+ start = name.start_with?("::") ? 2 : 0
383
+ separator = name.rindex("::")
384
+ cut = separator && separator >= start ? separator + 2 : start
385
+ cut == name.bytesize ? nil : name.byteslice(cut, name.bytesize - cut)
386
+ end
387
+ private_class_method :last_segment
388
+
389
+ def plain_constant_path?(name)
390
+ name.ascii_only? && !name.end_with?(":") && !name.include?(":::")
391
+ end
392
+ private_class_method :plain_constant_path?
393
+
213
394
  # The miss-only spelling of {read_name}: a cross-file lookup of `name` (kind `:method` / `:class` /
214
395
  # `:toplevel` / …) that resolved to nothing. Kept as the name the internal spec's negative-edge
215
396
  # contract uses.
@@ -217,13 +398,14 @@ module Rigor
217
398
  read_name(kind, name)
218
399
  end
219
400
 
220
- # The positive-edge aggregation shared by {read_site} and {replay}.
401
+ # The positive-edge aggregation shared by {read_site} and {replay}. `sources` is the union of the other
402
+ # two tables' paths, and this is the only writer of all three, so a read the finer table already holds is
403
+ # already in `sources` and costs one Set probe instead of two.
221
404
  def accumulate_read(accumulator, path, symbol)
222
- accumulator.sources << path
223
405
  if symbol
224
- accumulator.symbol_sources[path] << symbol
225
- else
226
- accumulator.ancestry_sources << path
406
+ accumulator.sources << path if accumulator.symbol_sources[path].add?(symbol)
407
+ elsif accumulator.ancestry_sources.add?(path)
408
+ accumulator.sources << path
227
409
  end
228
410
  end
229
411
  private_class_method :accumulate_read
@@ -0,0 +1,266 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "securerandom"
5
+
6
+ require_relative "../incremental_run_slot"
7
+ require_relative "../../cache/file_digest"
8
+ require_relative "../../cache/incremental_snapshot"
9
+
10
+ module Rigor
11
+ module Analysis
12
+ module IncrementalRunSlot
13
+ # ADR-45 WD2 (#1507) — whether a run may record its answer at all. Some of a slot's rows are built when the
14
+ # run ENDS rather than when it reads, and each of those must still describe the tree the run analysed: a
15
+ # save that lands while the run reads — an editor's, a `bundle install`'s, a `mkdir` — would otherwise leave
16
+ # the slot vouching for a tree the analysis never saw, and the probe would serve the pre-save answer against
17
+ # the post-save tree.
18
+ #
19
+ # So the session takes a mark before the run reads anything ({.start}) and asks {#admits?} about those rows
20
+ # before it writes. A row recorded as the run read (a plugin's read, a listing it took) needs no guard: a
21
+ # later save leaves it stale, and validation says so.
22
+ #
23
+ # The mark is the filesystem's own clock, read off the change time of a file written for the purpose: a
24
+ # timestamp compared across clocks misses an edit that lands inside a coarse filesystem's tick, and change
25
+ # times (unlike modification times) cannot be set back by `cp -p` or `touch -d`. One clock means one
26
+ # filesystem, so the mark is taken only when the store is on the project's filesystem, and a row the key
27
+ # does not pin refuses the write when it is on any other. Declining to write is always safe; it leaves the
28
+ # next null run on the full path.
29
+ #
30
+ # A filesystem's change times tick coarsely (a millisecond or more on ext4 and tmpfs), so a file saved just
31
+ # before the stamp can carry the stamp's own change time. {#admits?} refuses a change time at or after the
32
+ # mark, since a save after the stamp in the same tick reads the same; taking the first stamp as the mark
33
+ # would therefore refuse every save that landed shortly before the run, and no slot would be written. So the
34
+ # mark is taken on a tick boundary: stamps are written until one carries a later change time than the first,
35
+ # and that later change time is the mark. Everything saved before the run started carries a change time no
36
+ # later than the first stamp, hence before the mark; everything saved after the mark carries at least the
37
+ # mark. The wait is bounded ({TICK_WAIT_LIMIT}); a filesystem that does not tick within it takes no mark,
38
+ # which is the safe answer. A filesystem whose first stamp lands on a whole second (HFS+, FAT, ext3) ticks too
39
+ # coarsely to wait out and takes no mark at once.
40
+ #
41
+ # Where the change time is not the time of the write, the mark says nothing, and the guard takes none or
42
+ # cannot tell: native Windows, where a change time is the creation time, takes no mark; a network or FUSE
43
+ # filesystem (NFS with its attribute cache, virtiofs, sshfs) may report a change time from before a save,
44
+ # which nothing here detects. A clock that steps backwards (an NTP correction) inside the run would date a
45
+ # save before the mark, so {#admits?} takes one more stamp and refuses when it reads earlier than the mark.
46
+ # A step undone before the run ends is not seen.
47
+ class WriteGuard
48
+ STAMP_DIR = "incremental"
49
+ private_constant :STAMP_DIR
50
+
51
+ # How long to wait, in seconds, for the filesystem's change time to tick past the first stamp.
52
+ TICK_WAIT_LIMIT = 0.05
53
+ # The pause between stamps.
54
+ TICK_RETRY_DELAY = 0.0005
55
+ private_constant :TICK_RETRY_DELAY
56
+
57
+ # Takes the mark, or nil when the slot must not be written: no stamp can be written, it is not on the
58
+ # project's filesystem, or the snapshot fingerprint the run was given no longer describes the tree.
59
+ #
60
+ # The fingerprint is recomputed AFTER the stamp: the caller computed it earlier, and a lockfile or a
61
+ # signature file under the configured or auto-detected `sig/` roots, or a `pre_eval:` file, changed in
62
+ # between would leave the snapshot the run restores keyed by one tree
63
+ # and the run reading another. From the stamp on, the lockfiles are the guard's to watch ({#admits?}), which
64
+ # is also what lets the slot's key, whose only file inputs they are, be computed when the run ends.
65
+ #
66
+ # The mark also records which of the paths an existence row can name are present: the lockfiles, the
67
+ # analysis roots, the `pre_eval:` entries and the signature roots.
68
+ def self.start(configuration:, roots:, cache_root:, fingerprint:)
69
+ return nil if Gem.win_platform?
70
+
71
+ started_ns, device = stamp(cache_root)
72
+ return nil if started_ns.nil? || device_of(Dir.pwd) != device
73
+ return nil unless Cache::IncrementalSnapshot.fingerprint(configuration: configuration, roots: roots) ==
74
+ fingerprint
75
+
76
+ lockfiles = lockfile_paths(configuration)
77
+ named = Array(roots) + configuration.pre_eval + (configuration.signature_paths || ["sig"])
78
+ presence = (lockfiles + named.map { |path| File.absolute_path(path.to_s) }).to_h do |path|
79
+ [path, File.exist?(path)]
80
+ end
81
+ new(started_ns: started_ns, device: device, lockfiles: lockfiles, presence: presence,
82
+ stamp_dir: stamp_dir(cache_root))
83
+ end
84
+
85
+ # The mark: the change time and device of the first stamp whose change time is later than that of a
86
+ # stamp taken before it, on the same device; nil when none is written within {TICK_WAIT_LIMIT}.
87
+ def self.stamp(cache_root)
88
+ dir = stamp_dir(cache_root)
89
+ FileUtils.mkdir_p(dir)
90
+ first_ns, device = write_stamp(dir)
91
+ # A change time on a whole second is a filesystem that ticks by the second: waiting for it would burn the
92
+ # whole bound. One that ticks finer lands there once in a billion stamps, and declining is safe.
93
+ return nil if first_ns.nil? || (first_ns % 1_000_000_000).zero?
94
+
95
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TICK_WAIT_LIMIT
96
+ loop do
97
+ ns, dev = write_stamp(dir)
98
+ return nil if ns.nil? || dev != device
99
+ return [ns, device] if ns > first_ns
100
+ return nil if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
101
+
102
+ sleep(TICK_RETRY_DELAY)
103
+ end
104
+ rescue SystemCallError, IOError
105
+ nil
106
+ end
107
+ private_class_method :stamp
108
+
109
+ def self.stamp_dir(cache_root)
110
+ File.join(cache_root.to_s, STAMP_DIR)
111
+ end
112
+ private_class_method :stamp_dir
113
+
114
+ # One stamp's change time and device; the file is removed again.
115
+ def self.write_stamp(dir)
116
+ path = File.join(dir, "run-#{Process.pid}-#{SecureRandom.hex(4)}.stamp")
117
+ File.write(path, "")
118
+ stat = File.stat(path)
119
+ [Cache::FileDigest.ns_of(stat.ctime), stat.dev]
120
+ rescue SystemCallError, IOError
121
+ nil
122
+ ensure
123
+ FileUtils.rm_f(path) if path
124
+ end
125
+ private_class_method :write_stamp
126
+
127
+ def self.device_of(path)
128
+ File.stat(path).dev
129
+ rescue SystemCallError
130
+ nil
131
+ end
132
+ private_class_method :device_of
133
+
134
+ # Every lockfile the key or the snapshot fingerprint may digest, present or not: a lockfile that appears
135
+ # while the run reads moves the key as surely as one rewritten, and one rewritten and restored digests as it
136
+ # did while the run may have read the other bytes.
137
+ def self.lockfile_paths(configuration)
138
+ configured = [
139
+ [configuration.bundler_lockfile, Environment::LockfileResolver],
140
+ [configuration.rbs_collection_lockfile, Environment::RbsCollectionDiscovery]
141
+ ].filter_map { |path, resolver| resolver.configured_lockfile_path(path) if path }
142
+ (configured + %w[Gemfile.lock rbs_collection.lock.yaml].map { |path| File.absolute_path(path) }).uniq
143
+ end
144
+ private_class_method :lockfile_paths
145
+
146
+ def initialize(started_ns:, device:, lockfiles:, presence: {}, stamp_dir: nil)
147
+ @stamp_dir = stamp_dir
148
+ @started_ns = started_ns
149
+ @device = device
150
+ @lockfiles = lockfiles
151
+ @presence = presence
152
+ end
153
+
154
+ # True when nothing the guard watches moved since the mark:
155
+ #
156
+ # - no lockfile the key or the fingerprint digests was written, created or removed;
157
+ # - no file `rows` or `pinned` names changed. A content row's file changed if its change time moved. An
158
+ # existence row asks only whether its path is there, so a path the mark recorded changed if it came or
159
+ # went, and an editor's lock file created and removed beside it, which moves its directory's change
160
+ # time, does not count. That needs no clock, so it holds on any filesystem; it cannot see a path that
161
+ # comes and goes again within the run. Any other existence row falls back to change times: its own when
162
+ # present, its nearest existing ancestor's when absent;
163
+ # - no directory a glob row lists gained or lost an entry, nor, for a stat-mode glob, did any file it
164
+ # matches change.
165
+ #
166
+ # A path on another filesystem than the mark refuses the write when it is one of `rows`, and is passed
167
+ # over when it is one of `pinned`: the rows the key pins (the engine's own signature files, a gem's
168
+ # signatures, which the engine slot and the lockfiles identify) live wherever the engine and the gems are
169
+ # installed — a container image's layer, a Nix store — and refusing them would turn the fast path off for
170
+ # every such installation. On the mark's filesystem they are checked like any other row.
171
+ #
172
+ # A content row whose file is gone passes: the row is stale already, and validation says so.
173
+ #
174
+ # @param rows — a {Cache::Descriptor} of the rows the key does not pin
175
+ # @param pinned — a {Cache::Descriptor} of the rows it does
176
+ def admits?(rows, pinned: Cache::Descriptor.new)
177
+ !clock_stepped_back? && @lockfiles.none? { |path| lockfile_changed?(path) } &&
178
+ files_unchanged?(rows.files, strict: true) && globs_unchanged?(rows.globs, strict: true) &&
179
+ files_unchanged?(pinned.files, strict: false) && globs_unchanged?(pinned.globs, strict: false)
180
+ rescue StandardError
181
+ false
182
+ end
183
+
184
+ # `descriptor` with every `:stat` row's recording instant no later than the mark. A row recorded as the run
185
+ # read carries the instant the run began, and validation trusts a stat tuple whose mtime is before it. On a
186
+ # coarse clock a save that lands after the read, in the tick the file's mtime already has and with the same
187
+ # size, leaves the tuple unmoved, so it would be trusted. With the instant at the mark, every row whose mtime is
188
+ # at or after the mark is racy, and validation re-hashes it.
189
+ def mark_racy(descriptor)
190
+ Cache::Descriptor.new(
191
+ files: descriptor.files.map { |entry| entry.with_recording_instant_at_most(@started_ns) },
192
+ gems: descriptor.gems, plugins: descriptor.plugins, configs: descriptor.configs,
193
+ dependencies: descriptor.dependencies, globs: descriptor.globs
194
+ )
195
+ end
196
+
197
+ private
198
+
199
+ # One more stamp: a change time earlier than the mark means the clock stepped back since it was taken, so
200
+ # a save since then may carry a change time the guard reads as before it. A stamp that cannot be written,
201
+ # or lands on another device, cannot vouch for the clock either.
202
+ def clock_stepped_back?
203
+ return false if @stamp_dir.nil?
204
+
205
+ ns, device = self.class.__send__(:write_stamp, @stamp_dir)
206
+ ns.nil? || device != @device || ns < @started_ns
207
+ end
208
+
209
+ # An existence row and a content row for one path are different questions (the first asks only whether it
210
+ # is there, the second whether it changed), so the row that comes first must not hide the other.
211
+ def files_unchanged?(entries, strict:)
212
+ seen = Set.new
213
+ entries.none? do |entry|
214
+ next false unless seen.add?([entry.path, entry.comparator == :exists])
215
+
216
+ if entry.comparator == :exists
217
+ existence_changed?(entry.path, strict: strict)
218
+ else
219
+ changed?(entry.path, strict: strict)
220
+ end
221
+ end
222
+ end
223
+
224
+ def globs_unchanged?(entries, strict:)
225
+ entries.none? do |entry|
226
+ listed = Dir.glob(File.join(entry.root, File.dirname(entry.pattern), ""))
227
+ matched = entry.mode == :stat ? Dir.glob(File.join(entry.root, entry.pattern)) : []
228
+ (listed + matched).any? { |path| changed?(path, strict: strict) }
229
+ end
230
+ end
231
+
232
+ # A lockfile is read for its bytes: one absent at the mark must still be absent, one present must still be
233
+ # there with its change time before the mark.
234
+ def lockfile_changed?(path)
235
+ return File.exist?(path) unless @presence[path]
236
+
237
+ changed?(path, strict: true, missing: true)
238
+ end
239
+
240
+ def existence_changed?(path, strict:)
241
+ absolute = File.absolute_path(path)
242
+ return File.exist?(absolute) != @presence[absolute] if @presence.key?(absolute)
243
+
244
+ nearest = absolute
245
+ nearest = File.dirname(nearest) until File.exist?(nearest) || File.dirname(nearest) == nearest
246
+ moved?(File.stat(nearest), strict: strict)
247
+ rescue SystemCallError
248
+ true
249
+ end
250
+
251
+ def changed?(path, strict:, missing: false)
252
+ moved?(File.stat(path), strict: strict)
253
+ rescue SystemCallError
254
+ missing
255
+ end
256
+
257
+ # A change time is comparable with the mark only on the stamp's filesystem.
258
+ def moved?(stat, strict:)
259
+ return strict if stat.dev != @device
260
+
261
+ Cache::FileDigest.ns_of(stat.ctime) >= @started_ns
262
+ end
263
+ end
264
+ end
265
+ end
266
+ end