rspec-tracer 1.2.3 → 2.0.0.pre.2

data/lib/rspec_tracer/storage/lazy_snapshot.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative 'snapshot'
+
+module RSpecTracer
+  module Storage
+    # Lazy-reading view returned by `JsonBackend#load_graph`. Presents
+    # the same field surface as `Storage::Snapshot` but defers the disk
+    # read + deserialization for each field until the caller touches it.
+    #
+    # Rationale: on a large cache (100 MB+), eager-reading every file
+    # at setup dominates warm-run startup (issue #17). Engine
+    # at setup touches 10/15 fields; reporters never touch the previous
+    # snapshot; third-party tooling often reads one or two. The 3-5
+    # fields the Engine does NOT touch at setup (duplicate_examples,
+    # reverse_dependency, env_dependency; examples_coverage is deferred
+    # to finalize per the seed refactor) save their full disk+parse
+    # cost for every run.
+    #
+    # Duck-types `Snapshot`: every Struct member becomes a method here,
+    # `respond_to?` returns true for each, `to_h` materializes the
+    # whole thing. Callers doing `prev.send(field)` or `prev.field`
+    # work unchanged.
+    #
+    # Thread-safety: the memoization Hash is not guarded. Engine is
+    # single-threaded per run; parallel_tests workers each own their
+    # own LazySnapshot instance. If a future caller reads fields from
+    # multiple threads, wrap with a Monitor at construct time.
+    class LazySnapshot
+      LAZY_FIELDS = (Snapshot.members - %i[schema_version run_id]).freeze
+
+      attr_reader :schema_version, :run_id
+
+      def initialize(schema_version:, run_id:, reader:)
+        @schema_version = schema_version
+        @run_id = run_id
+        @reader = reader
+        @loaded = {}
+      end
+
+      LAZY_FIELDS.each do |field|
+        define_method(field) do
+          return @loaded[field] if @loaded.key?(field)
+
+          @loaded[field] = @reader.read(field)
+        end
+      end
+
+      # Hash view matching `Struct#to_h` so callers composing snapshots
+      # (merge pipelines, reporter helpers) get the familiar shape.
+      # Forces every field to materialize, so only call this when the
+      # full cache is genuinely required.
+      def to_h
+        Snapshot.members.to_h { |m| [m, public_send(m)] }
+      end
+
+      # Materialize a full eager Snapshot. Use when an API requires the
+      # Struct type (Merger input, backends that accept Snapshot on
+      # save). Reads every field.
+      def to_snapshot
+        Snapshot.new(**to_h)
+      end
+    end
+  end
+end

data/lib/rspec_tracer/storage/schema.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  # Internal Storage — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module Storage
+    # Cache schema version and compatibility policy. 1.x shipped caches
+    # without any version stamp; 2.0's first migration is simply to
+    # start emitting a version and refuse to load anything else.
+    #
+    # Compatibility rule (from ARCHITECTURE.md):
+    #   - `CURRENT` is written into every new cache manifest.
+    #   - `SUPPORTED` is the set of versions this backend is willing
+    #     to load. For 2.0 there is exactly one supported version.
+    #   - On mismatch, `load_graph` returns nil after an `info` log
+    #     line and the run proceeds cold. No in-place migrators.
+    #
+    # Future version bumps (schema_version 3, 4, ...) add entries to
+    # `SUPPORTED` only if the backend can load both shapes. If a
+    # change is breaking, `SUPPORTED` resets to `[CURRENT]` and the
+    # caller pays one cold run on upgrade - the deal 1.x users already
+    # expect for any rspec-tracer version bump.
+    module Schema
+      # 1.x caches were unstamped; schema_version 2 was the first
+      # versioned schema, and 2.0 bumped to 3 when `Snapshot.boot_set`
+      # landed. schema_version 4 reshaped the example-identity payload:
+      # `example_id` hashes the describe block's *description* (not
+      # RSpec's load-order-dependent class name) and excludes line
+      # numbers. schema_version 5 closes the remaining gap for unnamed
+      # examples (`it { }` / `specify { }` / `example { }`): their
+      # `example_id` now derives from an intra-group ordinal instead
+      # of RSpec's line-bearing `"example at <path>:<line>"` fallback,
+      # so a 4-stamped cache's unnamed-example ids no longer match.
+      # Each bump is breaking, so SUPPORTED stays `[CURRENT]` and the
+      # caller pays one cold run on upgrade.
+      CURRENT = 5
+      # Internal constant.
+      # @api private
+      SUPPORTED = [CURRENT].freeze
+
+      # True when the caller can load a cache stamped with `version`.
+      # nil (an unstamped 1.x cache) is explicitly unsupported -
+      # treating nil as "compatible" would defeat the whole point of
+      # the version field. The caller logs and falls back to cold run.
+      def self.supported?(version)
+        SUPPORTED.include?(version)
+      end
+    end
+  end
+end

data/lib/rspec_tracer/storage/serializer/json.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RSpecTracer
+  # Internal Storage — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module Storage
+    # Internal Serializer — see {RSpecTracer} for the user-facing surface.
+    # @api private
+    module Serializer
+      # Default serializer for JsonBackend. Produces pretty-printed
+      # JSON strings; reads tolerate binary-mode bytes by forcing
+      # UTF-8 on decode (preserves the fix for example titles with
+      # non-ASCII bytes on US-ASCII-defaulted filesystems).
+      #
+      # Class-level methods (not module_function) so mutant-rspec can
+      # observe mutations through the call path; see the mutation-
+      # friendly-modules memo.
+      class Json
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.extension
+          'json'
+        end
+
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.encode(payload)
+          ::JSON.pretty_generate(payload)
+        end
+
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.decode(bytes)
+          ::JSON.parse(bytes.dup.force_encoding('UTF-8'))
+        end
+      end
+    end
+  end
+end

data/lib/rspec_tracer/storage/serializer/msgpack.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module RSpecTracer
+  # Internal Storage — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module Storage
+    # Internal Serializer — see {RSpecTracer} for the user-facing surface.
+    # @api private
+    module Serializer
+      # Raised when the caller asks for :msgpack but the msgpack gem
+      # is not in the user's bundle. JsonBackend rescues at construct
+      # time + falls back to the Json serializer with a warn line,
+      # same optional-dep pattern the RedisBackend uses for the redis
+      # gem.
+      class MsgpackGemNotInstalled < StandardError; end
+
+      # MessagePack + zlib serializer. msgpack encode is ~2x smaller
+      # than pretty JSON on our representative dependency graphs;
+      # zlib deflate then buys another ~5x on path-repetitive
+      # payloads (dependency.json is the big one - the same path
+      # string appears once per example that touched it). Combined
+      # ratio matches the ~4-6x claim in remote_cache/archive.rb.
+      #
+      # Stdlib zlib avoids adding a second gem dep; msgpack itself is
+      # the single new dev-group gem. Users who want the backend
+      # add `gem 'msgpack'` to their own Gemfile per
+      # USER_FACING_SURFACE.md optional-dep convention.
+      #
+      # The require is lazy so pure-Ruby suites that stay on :json
+      # do not pay the msgpack load cost and so the LoadError path
+      # is exercisable in unit specs (hide_const-based).
+      #
+      # Type extensions: Ruby `Time` and `Symbol` values surface in
+      # example metadata (RSpec's `execution_result.started_at` is a
+      # Time; status values like `:passed` / `:failed` / `:flaky` are
+      # Symbols on the in-memory snapshot). The bare msgpack default
+      # registry has no `Time` packer (crashes
+      # `NoMethodError: undefined method 'to_msgpack'`) and silently
+      # coerces `Symbol` to `String` (lossy round-trip). Both
+      # behaviors broke the cache on the first run a user followed
+      # the 50 MiB warning's `:msgpack` recommendation. Registering
+      # `Factory` type extensions for `Time` (ID 0x00, 12-byte
+      # seconds+nanoseconds payload) and `Symbol` (ID 0x01, UTF-8
+      # string payload) gives lossless round-trip. The factory is
+      # memoized so the extension registration is one-shot per
+      # process.
+      class Msgpack
+        # Internal constant — `MessagePack::Factory#register_type`
+        # ID for Ruby `Time`. Codepoint chosen from the user-space
+        # range (0x00–0x7F per msgpack ext spec); collision-safe with
+        # the built-in timestamp ext (ID -1 / 0xFF) since IDs are
+        # disjoint.
+        # @api private
+        TIME_EXTENSION_TYPE = 0x00
+        # Internal constant — `MessagePack::Factory#register_type` ID
+        # for Ruby `Symbol`. See {TIME_EXTENSION_TYPE} rationale.
+        # @api private
+        SYMBOL_EXTENSION_TYPE = 0x01
+
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.extension
+          'msgpack.gz'
+        end
+
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.encode(payload)
+          ::Zlib::Deflate.deflate(factory.pack(payload))
+        end
+
+        # Internal helper for the tracer pipeline.
+        # @api private
+        def self.decode(bytes)
+          factory.unpack(::Zlib::Inflate.inflate(bytes))
+        end
+
+        # Probe used by JsonBackend#initialize to decide whether to
+        # fall back to the Json serializer at construct time. True
+        # iff `require 'msgpack'` succeeds; false when the gem is
+        # missing. Idempotent without an explicit memo because Ruby's
+        # `require` short-circuits via `$LOADED_FEATURES` on repeat
+        # calls and `defined?(::MessagePack)` cheaply detects the
+        # post-load constant. The previous @msgpack_loaded ivar memo
+        # was load-bearing for mutation observability: once any prior
+        # test tripped the memo, mutations on the require line were
+        # observably equivalent.
+        def self.available?
+          ensure_available!
+          true
+        rescue MsgpackGemNotInstalled
+          false
+        end
+
+        class << self
+          private
+
+          # `MessagePack::Factory` with `Time` and `Symbol` type
+          # extensions registered. Memoized for the process lifetime;
+          # the factory's `register_type` calls are not idempotent
+          # (re-registering would double-register the codepoint), so
+          # one-shot construction is the simplest correctness
+          # contract.
+          def factory
+            return @factory if defined?(@factory) && @factory
+
+            ensure_available!
+            @factory = build_factory
+          end
+
+          # `Time` and `Symbol` packers / unpackers. Packers are
+          # called by `factory.pack` whenever a Ruby `Time` /
+          # `Symbol` appears at any depth in the payload (top-level
+          # value, nested Hash value, Array element). Unpackers are
+          # called by `factory.unpack` on the ext-typed bytes.
+          # Symbol pack/unpack maps to `to_s` / `to_sym` directly
+          # via `Symbol#to_proc`; Time is more involved so it gets
+          # named `pack_time` / `unpack_time` helpers below.
+          def build_factory
+            f = ::MessagePack::Factory.new
+            f.register_type(
+              TIME_EXTENSION_TYPE, ::Time,
+              packer: method(:pack_time),
+              unpacker: method(:unpack_time)
+            )
+            f.register_type(
+              SYMBOL_EXTENSION_TYPE, ::Symbol,
+              packer: :to_s.to_proc,
+              unpacker: :to_sym.to_proc
+            )
+            f
+          end
+
+          # Time encoding: 64-bit signed seconds (`tv_sec`) + 32-bit
+          # signed nanoseconds (`tv_nsec`), little-endian, 12 bytes
+          # total. Round-trip canonicalizes to UTC — rspec-tracer
+          # never re-uses the Time as a user-facing wall-clock value
+          # (cache entries are compared, not displayed), and UTC
+          # makes the on-disk bytes timezone-independent so a cache
+          # built in one tz reads back identically in another.
+          def pack_time(time)
+            [time.tv_sec, time.tv_nsec].pack('q<l<')
+          end
+
+          def unpack_time(data)
+            sec, nsec = data.unpack('q<l<')
+            ::Time.at(sec, nsec, :nanosecond).utc
+          end
+
+          # Internal method on the tracer pipeline.
+          # @api private
+          def ensure_available!
+            return if defined?(::MessagePack)
+
+            require 'msgpack'
+          rescue ::LoadError
+            raise MsgpackGemNotInstalled,
+                  "msgpack gem is not installed; add `gem 'msgpack'` to your Gemfile " \
+                  'to use the :msgpack serializer.'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec_tracer/storage/snapshot.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module RSpecTracer
+  # Internal Storage — see {RSpecTracer} for the user-facing surface.
+  # @api private
+  module Storage
+    # Value object returned by `Backend#load_graph` and accepted by
+    # `Backend#save_graph`. Bundles every collection that 1.x's
+    # report_writer persists plus the schema_version + run_id envelope
+    # so a full cache can be reconstructed in one trip.
+    #
+    # The field layout mirrors 1.x's on-disk contract (see
+    # `JsonBackend::FILENAMES`); every field round-trips through
+    # `to_h` / `from_h` so the backend can serialize without reaching
+    # into struct internals.
+    #
+    # `examples_coverage` may be `nil` when a caller explicitly loads
+    # the cheap header only. The default load is eager - `nil` vs `{}`
+    # distinguishes "not yet loaded" from "loaded and empty."
+    #
+    # Methods are defined on the reopened class body (not inside the
+    # Struct.new block) so mutant can introspect them - same pattern
+    # as Tracker::Input.
+    #
+    # `boot_set` - Hash[relative_path => sha256_hex] of every project
+    # file loaded before any example runs (spec_helper requires, gem
+    # boot, eager autoload). Backs the constants-blind-spot fix: the
+    # engine compares this against the previous run's boot_set and ORs
+    # any mismatch with WholeSuiteInvalidators when computing the
+    # whole_suite_invalidated bool. Per-example loaded-set attribution
+    # folds into `dependency` via the existing graph registration path
+    # - no separate field.
+    #
+    # `wsi_snapshot` - Hash[watch_name => sha256_hex] produced by
+    # `WholeSuiteInvalidators#digest_snapshot`. Without it, a warm run
+    # can't tell whether Gemfile.lock / .ruby-version / .rspec-tracer
+    # (or the tracer gem identity) changed since the previous run, and
+    # the engine falls back to "first run = invalidate everything" on
+    # every warm run. The field is optional in the JSON layout so older
+    # caches continue to load (missing wsi.json coerces to `{}`, which
+    # compares unequal and triggers one cold re-run - safe fallback,
+    # same cost as any other cache miss).
+    #
+    # `env_snapshot` - Hash[env_name => md5_hex] produced by
+    # `Tracker::EnvSnapshot#digest_snapshot`. Covers env-var values
+    # declared via the per-example `tracks: { env: ... }` DSL.
+    # Without it, a warm run can't tell whether an env-gated example
+    # needs to re-run when the env changes. Same optional-in-JSON
+    # treatment as wsi_snapshot: missing file coerces to `{}`, no
+    # schema_version bump, one cold re-run on upgrade.
+    #
+    # `env_dependency` - Hash[example_id => Array<env_name>] capturing
+    # which env keys each tracked example declared. The per-run
+    # `env_snapshot` stores the digest of each key; this map stores
+    # the example-to-key attribution that the reporter layer needs to
+    # render "which env vars does this example depend on." Without it,
+    # reports can't surface env dependencies - Engine's per-run
+    # `@tracks_env` map would otherwise be lost at finalize.
+    # Same optional-in-JSON treatment as wsi_snapshot / env_snapshot:
+    # missing file coerces to `{}`, no schema_version bump.
+    #
+    # `cache_hit_reason` is a SUITE-LEVEL aggregate. It maps
+    # `reason_string => count` (e.g. `{"Files changed" => 12,
+    # "No cache" => 5}`) and surfaces "why did each non-skipped
+    # example run." JsonBackend persists it as `cache_hit_reason.json`
+    # under the per-run dir (same shape as wsi_snapshot / env_snapshot:
+    # one file per field; missing file coerces to `{}`, no
+    # schema_version bump). SqliteBackend does not persist it (would
+    # require a meta-table column / schema bump); SqliteBackend users
+    # see the field as `{}` on read until a future enhancement
+    # extends the meta table.
+    #
+    # `filtered_examples` is the PER-EXAMPLE source-of-truth that
+    # backs `cache_hit_reason`: a Hash[example_id => reason_string]
+    # ("ex_abc" => "Failed previously"). The engine writes this at
+    # finalize; `cache_hit_reason` is the values-tally. Persisting
+    # both lets the parallel-tests merge collapse per-worker
+    # duplicates by id (every worker computes the same hash because
+    # the filter walks the global previous-run snapshot) and re-tally
+    # at merge time, instead of sum-merging identical per-worker
+    # tallies and inflating counts N-fold. Same optional-in-JSON
+    # treatment as cache_hit_reason: missing file coerces to `{}`,
+    # no schema_version bump, JSON-backend-only surface.
+    Snapshot = Struct.new(
+      :schema_version,
+      :run_id,
+      :all_examples,
+      :duplicate_examples,
+      :interrupted_examples,
+      :flaky_examples,
+      :failed_examples,
+      :pending_examples,
+      :skipped_examples,
+      :all_files,
+      :dependency,
+      :reverse_dependency,
+      :examples_coverage,
+      :boot_set,
+      :wsi_snapshot,
+      :env_snapshot,
+      :env_dependency,
+      :cache_hit_reason,
+      :filtered_examples,
+      keyword_init: true
+    )
+
+    # Internal Snapshot — see {RSpecTracer} for the user-facing surface.
+    # @api private
+    class Snapshot
+      # Defaults every collection to its 1.x starting shape: Hash for
+      # keyed collections, Set for example-id lists. Keeps spec
+      # construction terse and prevents accidental nil-deref when a
+      # save is composed incrementally.
+      def self.empty(schema_version:, run_id:)
+        new(
+          schema_version: schema_version,
+          run_id: run_id,
+          all_examples: {},
+          duplicate_examples: {},
+          interrupted_examples: Set.new,
+          flaky_examples: Set.new,
+          failed_examples: Set.new,
+          pending_examples: Set.new,
+          skipped_examples: Set.new,
+          all_files: {},
+          dependency: {},
+          reverse_dependency: {},
+          examples_coverage: {},
+          boot_set: {},
+          wsi_snapshot: {},
+          env_snapshot: {},
+          env_dependency: {},
+          cache_hit_reason: {},
+          filtered_examples: {}
+        )
+      end
+    end
+  end
+end