rigortype 0.4.0 → 0.4.2

data/lib/rigor/analysis/incremental_run_slot/write_guard.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "securerandom"
+
+require_relative "../incremental_run_slot"
+require_relative "../dependency_source_inference/gem_resolver"
+require_relative "../../cache/file_digest"
+require_relative "../../cache/incremental_snapshot"
+
+module Rigor
+  module Analysis
+    module IncrementalRunSlot
+      # ADR-45 WD2 (#1507) — whether a run may record its answer at all. Some of a slot's rows are built when the
+      # run ENDS rather than when it reads, and each of those must still describe the tree the run analysed: a
+      # save that lands while the run reads — an editor's, a `bundle install`'s, a `mkdir` — would otherwise leave
+      # the slot vouching for a tree the analysis never saw, and the probe would serve the pre-save answer against
+      # the post-save tree.
+      #
+      # So the session takes a mark before the run reads anything ({.start}) and asks {#admits?} about those rows
+      # before it writes. A row recorded as the run read (a plugin's read, a listing it took) needs no guard: a
+      # later save leaves it stale, and validation says so.
+      #
+      # The mark is the filesystem's own clock, read off the change time of a file written for the purpose: a
+      # timestamp compared across clocks misses an edit that lands inside a coarse filesystem's tick, and change
+      # times (unlike modification times) cannot be set back by `cp -p` or `touch -d`. One clock means one
+      # filesystem, so the mark is taken only when the store is on the project's filesystem, and a row the key
+      # does not pin refuses the write when it is on any other. Declining to write is always safe; it leaves the
+      # next null run on the full path.
+      #
+      # A filesystem's change times tick coarsely (a millisecond or more on ext4 and tmpfs), so a file saved just
+      # before the stamp can carry the stamp's own change time. {#admits?} refuses a change time at or after the
+      # mark, since a save after the stamp in the same tick reads the same; taking the first stamp as the mark
+      # would therefore refuse every save that landed shortly before the run, and no slot would be written. So the
+      # mark is taken on a tick boundary: stamps are written until one carries a later change time than the first,
+      # and that later change time is the mark. Everything saved before the run started carries a change time no
+      # later than the first stamp, hence before the mark; everything saved after the mark carries at least the
+      # mark. The wait is bounded ({TICK_WAIT_LIMIT}); a filesystem that does not tick within it takes no mark,
+      # which is the safe answer. A filesystem whose first stamp lands on a whole second (HFS+, FAT, ext3) ticks too
+      # coarsely to wait out and takes no mark at once.
+      #
+      # Where the change time is not the time of the write, the mark says nothing, and the guard takes none or
+      # cannot tell: native Windows, where a change time is the creation time, takes no mark; a network or FUSE
+      # filesystem (NFS with its attribute cache, virtiofs, sshfs) may report a change time from before a save,
+      # which nothing here detects. A clock that steps backwards (an NTP correction) inside the run would date a
+      # save before the mark, so {#admits?} takes one more stamp and refuses when it reads earlier than the mark.
+      # A step undone before the run ends is not seen.
+      class WriteGuard
+        STAMP_DIR = "incremental"
+        private_constant :STAMP_DIR
+
+        # How long to wait, in seconds, for the filesystem's change time to tick past the first stamp.
+        TICK_WAIT_LIMIT = 0.05
+        # The pause between stamps.
+        TICK_RETRY_DELAY = 0.0005
+        private_constant :TICK_RETRY_DELAY
+
+        # Takes the mark, or nil when the slot must not be written: no stamp can be written, it is not on the
+        # project's filesystem, or the snapshot fingerprint the run was given no longer describes the tree.
+        #
+        # The fingerprint is recomputed AFTER the stamp: the caller computed it earlier, and a lockfile or a
+        # signature file under the configured or auto-detected `sig/` roots, or a `pre_eval:` file, changed in
+        # between would leave the snapshot the run restores keyed by one tree
+        # and the run reading another. From the stamp on, the lockfiles are the guard's to watch ({#admits?}), which
+        # is also what lets the slot's key, whose only file inputs they are, be computed when the run ends. The
+        # opted-in gems' resolved versions, the key's one input that is neither a file nor the configuration (issue
+        # #1814), are read at the mark and watched the same way.
+        #
+        # The mark also records which of the paths an existence row can name are present: the lockfiles, the
+        # analysis roots, the `pre_eval:` entries and the signature roots.
+        def self.start(configuration:, roots:, cache_root:, fingerprint:)
+          return nil if Gem.win_platform?
+
+          started_ns, device = stamp(cache_root)
+          return nil if started_ns.nil? || device_of(Dir.pwd) != device
+          return nil unless Cache::IncrementalSnapshot.fingerprint(configuration: configuration, roots: roots) ==
+                            fingerprint
+
+          lockfiles = lockfile_paths(configuration)
+          named = Array(roots) + configuration.pre_eval + (configuration.signature_paths || ["sig"])
+          presence = (lockfiles + named.map { |path| File.absolute_path(path.to_s) }).to_h do |path|
+            [path, File.exist?(path)]
+          end
+          rows = dependency_rows(configuration)
+          return nil if rows.nil?
+
+          new(started_ns: started_ns, device: device, lockfiles: lockfiles, presence: presence,
+              stamp_dir: stamp_dir(cache_root), configuration: configuration, dependency_rows: rows)
+        end
+
+        # Issue #1814 — the opted-in gems' `(name, version, mode)` rows as the key carries them; nil when they cannot
+        # be resolved, which takes no mark (the key could not be computed either).
+        def self.dependency_rows(configuration)
+          DependencySourceInference::GemResolver.dependency_entries(configuration.dependencies)
+        rescue StandardError
+          nil
+        end
+        private_class_method :dependency_rows
+
+        # The mark: the change time and device of the first stamp whose change time is later than that of a
+        # stamp taken before it, on the same device; nil when none is written within {TICK_WAIT_LIMIT}.
+        def self.stamp(cache_root)
+          dir = stamp_dir(cache_root)
+          FileUtils.mkdir_p(dir)
+          first_ns, device = write_stamp(dir)
+          # A change time on a whole second is a filesystem that ticks by the second: waiting for it would burn the
+          # whole bound. One that ticks finer lands there once in a billion stamps, and declining is safe.
+          return nil if first_ns.nil? || (first_ns % 1_000_000_000).zero?
+
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TICK_WAIT_LIMIT
+          loop do
+            ns, dev = write_stamp(dir)
+            return nil if ns.nil? || dev != device
+            return [ns, device] if ns > first_ns
+            return nil if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+            sleep(TICK_RETRY_DELAY)
+          end
+        rescue SystemCallError, IOError
+          nil
+        end
+        private_class_method :stamp
+
+        def self.stamp_dir(cache_root)
+          File.join(cache_root.to_s, STAMP_DIR)
+        end
+        private_class_method :stamp_dir
+
+        # One stamp's change time and device; the file is removed again.
+        def self.write_stamp(dir)
+          path = File.join(dir, "run-#{Process.pid}-#{SecureRandom.hex(4)}.stamp")
+          File.write(path, "")
+          stat = File.stat(path)
+          [Cache::FileDigest.ns_of(stat.ctime), stat.dev]
+        rescue SystemCallError, IOError
+          nil
+        ensure
+          FileUtils.rm_f(path) if path
+        end
+        private_class_method :write_stamp
+
+        def self.device_of(path)
+          File.stat(path).dev
+        rescue SystemCallError
+          nil
+        end
+        private_class_method :device_of
+
+        # Every lockfile the key or the snapshot fingerprint may digest, present or not: a lockfile that appears
+        # while the run reads moves the key as surely as one rewritten, and one rewritten and restored digests as it
+        # did while the run may have read the other bytes.
+        def self.lockfile_paths(configuration)
+          configured = [
+            [configuration.bundler_lockfile, Environment::LockfileResolver],
+            [configuration.rbs_collection_lockfile, Environment::RbsCollectionDiscovery]
+          ].filter_map { |path, resolver| resolver.configured_lockfile_path(path) if path }
+          (configured + %w[Gemfile.lock rbs_collection.lock.yaml].map { |path| File.absolute_path(path) }).uniq
+        end
+        private_class_method :lockfile_paths
+
+        def initialize(started_ns:, device:, lockfiles:, presence: {}, stamp_dir: nil, configuration: nil,
+                       dependency_rows: [])
+          @stamp_dir = stamp_dir
+          @started_ns = started_ns
+          @device = device
+          @lockfiles = lockfiles
+          @presence = presence
+          @configuration = configuration
+          @dependency_rows = dependency_rows
+        end
+
+        # True when nothing the guard watches moved since the mark:
+        #
+        # - no lockfile the key or the fingerprint digests was written, created or removed, and no opted-in gem
+        #   resolves to another version than at the mark (issue #1814);
+        # - no file `rows` or `pinned` names changed. A content row's file changed if its change time moved. An
+        #   existence row asks only whether its path is there, so a path the mark recorded changed if it came or
+        #   went, and an editor's lock file created and removed beside it, which moves its directory's change
+        #   time, does not count. That needs no clock, so it holds on any filesystem; it cannot see a path that
+        #   comes and goes again within the run. Any other existence row falls back to change times: its own when
+        #   present, its nearest existing ancestor's when absent;
+        # - no directory a glob row lists gained or lost an entry, nor, for a stat-mode glob, did any file it
+        #   matches change.
+        #
+        # A path on another filesystem than the mark refuses the write when it is one of `rows`, and is passed
+        # over when it is one of `pinned`: the rows the key pins (the engine's own signature files, a gem's
+        # signatures, which the engine slot and the lockfiles identify) live wherever the engine and the gems are
+        # installed — a container image's layer, a Nix store — and refusing them would turn the fast path off for
+        # every such installation. On the mark's filesystem they are checked like any other row.
+        #
+        # A content row whose file is gone passes: the row is stale already, and validation says so.
+        #
+        # @param rows — a {Cache::Descriptor} of the rows the key does not pin
+        # @param pinned — a {Cache::Descriptor} of the rows it does
+        def admits?(rows, pinned: Cache::Descriptor.new)
+          !clock_stepped_back? && @lockfiles.none? { |path| lockfile_changed?(path) } && dependency_rows_unchanged? &&
+            files_unchanged?(rows.files, strict: true) && globs_unchanged?(rows.globs, strict: true) &&
+            files_unchanged?(pinned.files, strict: false) && globs_unchanged?(pinned.globs, strict: false)
+        rescue StandardError
+          false
+        end
+
+        # `descriptor` with every `:stat` row's recording instant no later than the mark. A row recorded as the run
+        # read carries the instant the run began, and validation trusts a stat tuple whose mtime is before it. On a
+        # coarse clock a save that lands after the read, in the tick the file's mtime already has and with the same
+        # size, leaves the tuple unmoved, so it would be trusted. With the instant at the mark, every row whose mtime is
+        # at or after the mark is racy, and validation re-hashes it.
+        def mark_racy(descriptor)
+          Cache::Descriptor.new(
+            files: descriptor.files.map { |entry| entry.with_recording_instant_at_most(@started_ns) },
+            gems: descriptor.gems, plugins: descriptor.plugins, configs: descriptor.configs,
+            dependencies: descriptor.dependencies, globs: descriptor.globs
+          )
+        end
+
+        private
+
+        # Issue #1814 — the opted-in gems resolve to the versions they did at the mark. A guard built without a
+        # configuration (the specs' bare constructor) watches none.
+        def dependency_rows_unchanged?
+          @configuration.nil? || self.class.send(:dependency_rows, @configuration) == @dependency_rows
+        end
+
+        # One more stamp: a change time earlier than the mark means the clock stepped back since it was taken, so
+        # a save since then may carry a change time the guard reads as before it. A stamp that cannot be written,
+        # or lands on another device, cannot vouch for the clock either.
+        def clock_stepped_back?
+          return false if @stamp_dir.nil?
+
+          ns, device = self.class.__send__(:write_stamp, @stamp_dir)
+          ns.nil? || device != @device || ns < @started_ns
+        end
+
+        # An existence row and a content row for one path are different questions (the first asks only whether it
+        # is there, the second whether it changed), so the row that comes first must not hide the other.
+        def files_unchanged?(entries, strict:)
+          seen = Set.new
+          entries.none? do |entry|
+            next false unless seen.add?([entry.path, entry.comparator == :exists])
+
+            if entry.comparator == :exists
+              existence_changed?(entry.path, strict: strict)
+            else
+              changed?(entry.path, strict: strict)
+            end
+          end
+        end
+
+        def globs_unchanged?(entries, strict:)
+          entries.none? do |entry|
+            listed = Dir.glob(File.join(entry.root, File.dirname(entry.pattern), ""))
+            matched = entry.mode == :stat ? Dir.glob(File.join(entry.root, entry.pattern)) : []
+            (listed + matched).any? { |path| changed?(path, strict: strict) }
+          end
+        end
+
+        # A lockfile is read for its bytes: one absent at the mark must still be absent, one present must still be
+        # there with its change time before the mark.
+        def lockfile_changed?(path)
+          return File.exist?(path) unless @presence[path]
+
+          changed?(path, strict: true, missing: true)
+        end
+
+        def existence_changed?(path, strict:)
+          absolute = File.absolute_path(path)
+          return File.exist?(absolute) != @presence[absolute] if @presence.key?(absolute)
+
+          nearest = absolute
+          nearest = File.dirname(nearest) until File.exist?(nearest) || File.dirname(nearest) == nearest
+          moved?(File.stat(nearest), strict: strict)
+        rescue SystemCallError
+          true
+        end
+
+        def changed?(path, strict:, missing: false)
+          moved?(File.stat(path), strict: strict)
+        rescue SystemCallError
+          missing
+        end
+
+        # A change time is comparable with the mark only on the stamp's filesystem.
+        def moved?(stat, strict:)
+          return strict if stat.dev != @device
+
+          Cache::FileDigest.ns_of(stat.ctime) >= @started_ns
+        end
+      end
+    end
+  end
+end

data/lib/rigor/analysis/incremental_run_slot.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require_relative "diagnostic"
+require_relative "path_expansion"
+require_relative "result"
+require_relative "run_cache_key"
+require_relative "../cache/descriptor"
+require_relative "../cache/file_digest"
+require_relative "../cache/store"
+
+module Rigor
+  module Analysis
+    # ADR-45 WD2 (#1507) — the run-result slot `rigor check --incremental` writes, and the engine-free probe that
+    # serves a null run from it. The writer is {IncrementalSession#run_incremental}; the reader is the CLI, before
+    # it loads the inference engine. Both go through this module, so the key, the producer id and the stored shape
+    # have one definition.
+    #
+    # It is ADR-45's record-and-validate slot with a second writer. The value is the diagnostics the session's run
+    # printed (before the baseline filter, which the CLI applies to every answer), and the dependency descriptor
+    # validates everything that answer was computed from:
+    #
+    # - one `:stat` row per analysed file, carrying the digest the SESSION holds for it — the bytes its cached rows
+    #   were computed from — rather than a re-digest taken after the run;
+    # - the rows the run re-read itself ({Runner#incremental_slot_rows}' `observed` and `derived` rows);
+    # - the chain the {Entry} carries forward: the rows the last full run recorded for the inputs the incremental
+    #   path does not re-derive on a recheck, and the plugin reads credited to each file's analysis
+    #   ({IncrementalSession#write_run_slot}).
+    #
+    # A hit is at least as careful as the full incremental path — where that path would notice a change, a row
+    # notices it first — but it is not a cold run: a stale answer that path serves for a read reaching a file
+    # through a plugin's memo (#1553) the probe may serve too. ADR-45 WD2 states the guarantee.
+    #
+    # Kept apart from the plain run's `analysis.run-diagnostics` by its own producer id and by the roots entry its
+    # key adds, either enough alone, so neither probe can ever read the other's entry, and an incremental defect
+    # cannot leak into a default run.
+    #
+    # The key is the one {RunCacheProbe} reconstructs from configuration alone — the library list without a
+    # `rbs.virtual_rbs` slot, and no `template-units` slot — plus the analysis roots ({Target}). Neither left-out
+    # slot is needed. A virtual RBS buffer is a function of an analysed file's bytes (a row) and of the
+    # synthesising plugin's identity and configuration (the key's `configuration`, lockfile and engine slots). A
+    # template unit is a function of its template's bytes and of what its transform read (rows) and of the
+    # plugin's identity and the engine's (key slots), so a project whose plugins claim template globs is served
+    # too. The ADR-87 probe misses on one: the plain slot's key carries the `template-units` slot.
+    #
+    # `explain:` is always false: the session's runners never set it (#1533), and the CLI declines the probe under
+    # `--explain`.
+    module IncrementalRunSlot
+      module_function
+
+      PRODUCER_ID = "analysis.incremental-run-diagnostics"
+
+      # One live generation per analysed-path set, as for the plain slot. A writer that moves the path set discards
+      # the entry it carried from (see {IncrementalSession}), so the churn a file addition makes is bounded without
+      # a compaction pass — the incremental path never runs one.
+      GENERATION_CAP = RunCacheKey::GENERATION_CAP
+
+      # The stored value. `baseline` and `pinned` (each a {Cache::Descriptor}) and `reads` (`{path =>
+      # Cache::Descriptor}`) are the chain the next writer carries forward, kept apart from the rest of the
+      # dependency descriptor so it knows which rows are which; their entries are the same objects the descriptor
+      # holds, so `Marshal` writes them once. `baseline` and `pinned` hold the rows the last full run recorded for
+      # inputs the incremental path does not re-derive on a recheck (the signature tree, the
+      # discovered-not-analysed files, the `pre_eval:` files outside the analysed set), `pinned` the signature
+      # files the key identifies already ({Runner::BaselineRows}); `reads` the plugin reads credited to each
+      # analysed file. `snapshot` is the identity of
+      # the incremental snapshot file the writing run left behind, which the next writer compares against the one
+      # it restored before trusting the chain. `roots` are the analysis roots as the run was given them
+      # ({.as_written}): the key holds them as a set of paths, and the probe serves the entry only to a run that
+      # names them the same way, in the same order.
+      Entry = Data.define(:diagnostics, :roots, :baseline, :pinned, :reads, :snapshot)
+
+      # What a slot is keyed by: the analysed-path set and the analysis roots the run was given. The roots are
+      # not implied by the files — `rigor check --incremental lib extra` with `extra` missing analyses the same
+      # files as `rigor check --incremental lib`, and only the first reports `extra` as missing.
+      Target = Data.define(:files, :roots)
+
+      # What a probe hit hands the CLI: the answer, and the analysed-file count its banner reports.
+      Hit = Data.define(:result, :file_count)
+
+      def key(configuration:, target:)
+        base = RunCacheKey.descriptor(
+          configuration: configuration, files: target.files, explain: false,
+          rbs_config_entries: RunCacheKey.libraries_config_entries(configuration)
+        )
+        return nil if base.nil?
+
+        roots = RunCacheKey.config_entry("incremental.roots", normalize_roots(target.roots).sort.join("\n"))
+        # Issue #1814 — the opted-in gems' `(name, version, mode)` rows travel with the key, as they do in the plain
+        # probe's: the gem's walked source reaches every file's answer and no analysed file records it.
+        Cache::Descriptor.new(gems: base.gems, dependencies: base.dependencies, configs: base.configs + [roots])
+      end
+
+      # The roots as absolute paths, which the KEY holds sorted: `lib`, `./lib` and `lib/` are one root there. A
+      # run that reorders its roots still finds the previous slot's chain (the snapshot fingerprint sorts them
+      # too), and one that respells them replaces the slot rather than adding one beside it.
+      def normalize_roots(roots)
+        Array(roots).map { |root| File.absolute_path(root.to_s) }
+      end
+
+      # The roots as the run was given them, which {Entry#roots} holds and {.serve} compares. A missing root is
+      # reported as written (`./extra: no such file or directory`), and `a b` lists `a`'s files before `b`'s, so
+      # neither a respelling nor a reordering may be served another's answer.
+      def as_written(roots)
+        Array(roots).map(&:to_s)
+      end
+
+      # The engine-free probe. Returns a {Hit} when the slot for this run's analysed-path set validates, or nil to
+      # decline: a miss, a stale row, an unreadable entry, a project with effect collection on (see below), or any
+      # failure at all — declining hands the run to the full path, which is always safe.
+      #
+      # Effect collection is declined here and never written by the session. Its configuration (`effects:`) is
+      # deliberately absent from `Configuration#to_h` and so from the key, and two of its rows are judged each run
+      # from state no engine-free path can rebuild (the plain probe's `envelope_lane_live?`, #428). With it off,
+      # the only effect row a run can print is `effect.annotations-unchecked`, which is a function of the
+      # signature tree and the analysed files' annotations — both validated rows — so it is served as stored.
+      def serve(configuration:, cache_root:, paths:)
+        return nil if configuration.effects_enabled?
+
+        files = PathExpansion.ruby_files(paths, configuration.exclude_patterns)
+        slot_key = key(configuration: configuration, target: Target.new(files: files, roots: paths))
+        return nil if slot_key.nil?
+
+        entry = validated_entry(configuration, cache_root, slot_key)
+        return nil unless entry.is_a?(Entry) && entry.diagnostics.is_a?(Array)
+        return nil unless entry.roots == as_written(paths)
+
+        Hit.new(result: Result.new(diagnostics: entry.diagnostics, stats: nil), file_count: files.size)
+      rescue StandardError
+        nil
+      end
+
+      # The previous slot's {Entry}, read WITHOUT validating it: it is stale by construction, since the run that
+      # is asking changed something. nil when there is no such slot, or it is not an {Entry} — the writer then
+      # skips its own write rather than guess what the files it served from cache depended on.
+      def previous_entry(store:, configuration:, target:)
+        slot_key = key(configuration: configuration, target: target)
+        return nil if slot_key.nil?
+
+        entry = store.peek_unvalidated(producer_id: PRODUCER_ID, key_descriptor: slot_key)
+        return nil unless entry.is_a?(Entry) && entry.reads.is_a?(Hash)
+        return nil unless entry.baseline.is_a?(Cache::Descriptor) && entry.pinned.is_a?(Cache::Descriptor)
+
+        entry
+      end
+
+      # Writes `entry` as the slot for `target`, validated by `dependencies` (the whole descriptor, of which the
+      # entry's chain is a part). Returns whether it wrote.
+      def write(store:, configuration:, target:, entry:, dependencies:)
+        slot_key = key(configuration: configuration, target: target)
+        return false if slot_key.nil?
+
+        store.store_validated(
+          producer_id: PRODUCER_ID, key_descriptor: slot_key, generation_cap: GENERATION_CAP,
+          value: entry, dependencies: dependencies
+        )
+      end
+
+      def discard(store:, configuration:, target:)
+        slot_key = key(configuration: configuration, target: target)
+        slot_key && store.discard(producer_id: PRODUCER_ID, key_descriptor: slot_key)
+      end
+
+      # Validation runs in its own per-run digest scope, with the configuration's `cache.validation` mode, exactly
+      # as the plain probe's does.
+      def validated_entry(configuration, cache_root, slot_key)
+        store = Cache::Store.new(root: cache_root, max_bytes: configuration.cache_max_bytes)
+        Cache::FileDigest.with_run(strict: configuration.cache_validation_strict?) do
+          store.peek_validated(producer_id: PRODUCER_ID, key_descriptor: slot_key)
+        end
+      end
+      private_class_method :validated_entry
+    end
+  end
+end