testsift 0.0.1

data/lib/testsift/rails/i18n_tracking.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require_relative '../tracker/file_digest'
+require_relative '../tracker/input'
+require_relative 'notifications'
+
+module TestSift
+  # Internal Rails — see {TestSift} for the user-facing surface.
+  # @api private
+  module Rails
+    # I18n backend observer. Covers custom backends (Redis-backed,
+    # DB-backed, Chain) that bypass YAML.load_file and would otherwise
+    # miss the IOHooks YAML hook.
+    #
+    # Mechanism: Module#prepend onto ::I18n::Backend::Base - every
+    # backend subclass's load_translations resolves through the hook,
+    # even when the subclass overrides and super-calls Base. Backends
+    # that never super-call Base#load_translations fall through the
+    # hook, but the common backends (Simple, Chain, Cascade) all do.
+    #
+    # Shares Notifications' thread-local bucket so Engine.setup opens
+    # and clears one bucket per example that covers both observer
+    # families. Engine harvests `bucket.values` at example_finished.
+    #
+    # Graceful degradation:
+    # - install no-ops if ::I18n::Backend::Base is absent (tracer
+    #   boot survives even in weird I18n-free app graphs).
+    # - Every record call swallows StandardError (CLAUDE.md) - a
+    #   digest failure or bucket-shape surprise never propagates into
+    #   the user's test run.
+    class I18nTracking
+      class << self
+        # Internal attribute.
+        # @api private
+        attr_reader :root
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def install(root:, filter: ->(_path) { true })
+          @root = File.expand_path(root)
+          @root_prefix = "#{@root}/"
+          @filter = filter
+          @prepended = prepend_backend_hook
+
+          self
+        end
+
+        # Prepended modules cannot be removed from the ancestry chain
+        # (Ruby has no public API), mirroring IOHooks.uninstall. Every
+        # hook entry point fast-rejects on @root_prefix nil once
+        # install state clears, so post-uninstall the hook is a no-op.
+        def uninstall
+          @root = nil
+          @root_prefix = nil
+          @filter = nil
+          @prepended = false
+          self
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def installed?
+          !@root_prefix.nil?
+        end
+
+        # Called from LoadTranslationsHook for every filename passed
+        # to I18n::Backend::Base#load_translations. Array form keeps
+        # the hook a single call site.
+        def record_translations(filenames)
+          return nil if @root_prefix.nil?
+
+          Array(filenames).each { |path| record_translation(path) }
+          nil
+        end
+
+        # Pure-logic entry point. Same fast-reject ladder as
+        # Notifications#record_template. Emits :notification kind
+        # so the I18n source is distinguishable from template events
+        # in downstream reporters. The ladder is longer than rubocop's
+        # perceived-complexity threshold by design - each guard is a
+        # cheap fast-reject.
+        # rubocop:disable Metrics/PerceivedComplexity
+        def record_translation(path)
+          return nil if @root_prefix.nil?
+
+          bucket = Notifications.current_bucket
+          return nil if bucket.nil?
+          return nil unless path.is_a?(String) || path.respond_to?(:to_s)
+
+          path_str = path.to_s
+          return nil unless path_str.start_with?(@root_prefix)
+          return nil unless @filter.call(path_str)
+
+          identity = "notification:#{path_str[@root_prefix.length..]}"
+          return nil if bucket.key?(identity)
+
+          digest = Tracker::FileDigest.compute(path_str)
+          return nil if digest.nil?
+
+          bucket[identity] = Tracker::Input.for_file(
+            path: path_str, kind: :notification, digest: digest, root: @root
+          )
+        rescue StandardError
+          nil
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        private
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def prepend_backend_hook
+          return false unless defined?(::I18n::Backend::Base)
+
+          ::I18n::Backend::Base.prepend(LoadTranslationsHook)
+          true
+        rescue StandardError
+          false
+        end
+      end
+
+      # Prepended onto I18n::Backend::Base. Every subclass's
+      # load_translations ultimately resolves through here via super.
+      # Intercepts the filename list, delegates recording to the
+      # singleton, and forwards to the real implementation.
+      # @api private
+      module LoadTranslationsHook
+        # Internal method on the tracer pipeline.
+        # @api private
+        def load_translations(*filenames)
+          TestSift::Rails::I18nTracking.record_translations(filenames)
+          super
+        end
+      end
+    end
+  end
+end

data/lib/testsift/rails/notifications.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require_relative '../tracker/file_digest'
+require_relative '../tracker/input'
+
+module TestSift
+  # Internal Rails — see {TestSift} for the user-facing surface.
+  # @api private
+  module Rails
+    # ActiveSupport::Notifications observer for Rails-side inputs that
+    # Coverage and IOHooks can't see directly. Subscribes to:
+    #
+    #   - render_template.action_view   -> :template
+    #   - render_partial.action_view    -> :template
+    #   - render_collection.action_view -> :template (Rails 7.1+)
+    #   - sql.active_record             -> :notification
+    #       (on the first query per example, emits db/schema.rb +
+    #        db/structure.sql as inputs)
+    #
+    # Lifecycle mirrors IOHooks: Engine.setup installs, example_started
+    # opens a thread-local bucket, subscribers append, example_finished
+    # harvests and clears the bucket, Engine.finalize unsubscribes.
+    #
+    # Payload access is defensively permissive - Rails minors differ on
+    # symbol vs string keys; missing payload or nil identifier is
+    # swallowed. Errors inside subscribers never propagate (CLAUDE.md
+    # "graceful degradation").
+    #
+    # Precedence: an observed template path already covered by a user
+    # declared glob (e.g. the Preset :views glob) skips notification
+    # emission via the injected `filter:` callable. Matches IOHooks'
+    # declared-glob-precedence rule from ARCHITECTURE.md.
+    #
+    # The sql.active_record subscriber is only attached when the caller
+    # passes a non-empty `ar_schema_paths:` list. Engine resolves the
+    # list from the `track_ar_schema_notifications` opt-in DSL; absent
+    # that, the AR subscriber is never installed and the sql.active_record
+    # event stream is ignored.
+    class Notifications
+      # Internal constant.
+      # @api private
+      BUCKET_KEY = :testsift_rails_bucket
+      # Internal constant.
+      # @api private
+      AR_FLAG_KEY = :testsift_rails_ar_emitted
+
+      class << self
+        # Internal attribute.
+        # @api private
+        attr_reader :root
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def install(root:, filter: ->(_path) { true }, ar_schema_paths: [])
+          @root = File.expand_path(root)
+          @root_prefix = "#{@root}/"
+          @filter = filter
+          @ar_schema_inputs = build_schema_inputs(ar_schema_paths)
+          @handles = []
+
+          subscribe_render_template
+          subscribe_render_partial
+          subscribe_render_collection if render_collection_supported?
+          subscribe_sql_active_record if ar_enabled?
+
+          self
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def uninstall
+          (@handles || []).each { |handle| safely_unsubscribe(handle) }
+          @handles = nil
+          @root = nil
+          @root_prefix = nil
+          @filter = nil
+          @ar_schema_inputs = nil
+          self
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def installed?
+          !@root_prefix.nil?
+        end
+
+        # rubocop:disable Naming/AccessorMethodName
+        def set_bucket(bucket)
+          Thread.current[BUCKET_KEY] = bucket
+          Thread.current[AR_FLAG_KEY] = false
+        end
+        # rubocop:enable Naming/AccessorMethodName
+
+        def clear_bucket
+          Thread.current[BUCKET_KEY] = nil
+          Thread.current[AR_FLAG_KEY] = nil
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def current_bucket
+          Thread.current[BUCKET_KEY]
+        end
+
+        # Pure-logic entry point for the render_*.action_view
+        # subscribers. Extracted so mutation smoke can exercise
+        # payload parsing without driving AS::Notifications.
+        def handle_render_event(payload)
+          return nil unless payload.is_a?(Hash)
+
+          identifier = payload[:identifier] || payload['identifier']
+          return nil if identifier.nil?
+
+          record_template(identifier)
+        end
+
+        # Pure-logic entry point for the sql.active_record subscriber.
+        # Payload contents are ignored - the event itself is the
+        # "this example touched AR" signal.
+        def handle_sql_event(_payload)
+          record_ar_schema
+        end
+
+        # Record a :template Input for the observed render identifier.
+        # Guarded by the same fast-reject ladder as IOHooks: install
+        # state, bucket presence, path-under-root, filter callable,
+        # identity dedup. The early-return ladder looks complex to
+        # rubocop's metric but is the simplest shape for a hot path.
+        # rubocop:disable Metrics/PerceivedComplexity
+        def record_template(path)
+          return nil if @root_prefix.nil?
+
+          bucket = Thread.current[BUCKET_KEY]
+          return nil if bucket.nil?
+          return nil unless path.is_a?(String) || path.respond_to?(:to_s)
+
+          path_str = path.to_s
+          return nil unless path_str.start_with?(@root_prefix)
+          return nil unless @filter.call(path_str)
+
+          identity = "template:#{path_str[@root_prefix.length..]}"
+          return nil if bucket.key?(identity)
+
+          digest = Tracker::FileDigest.compute(path_str)
+          return nil if digest.nil?
+
+          bucket[identity] = Tracker::Input.for_file(
+            path: path_str, kind: :template, digest: digest, root: @root
+          )
+        rescue StandardError
+          nil
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        # Emit the pre-digested schema Inputs into the bucket on the
+        # first sql.active_record event per example. Subsequent events
+        # short-circuit via AR_FLAG_KEY - O(1) after the first query.
+        def record_ar_schema
+          return nil if @ar_schema_inputs.nil? || @ar_schema_inputs.empty?
+
+          bucket = Thread.current[BUCKET_KEY]
+          return nil if bucket.nil?
+          return nil if Thread.current[AR_FLAG_KEY]
+
+          Thread.current[AR_FLAG_KEY] = true
+          @ar_schema_inputs.each do |input|
+            bucket[input.identity] ||= input
+          end
+          nil
+        rescue StandardError
+          nil
+        end
+
+        private
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def ar_enabled?
+          !(@ar_schema_inputs.nil? || @ar_schema_inputs.empty?)
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def subscribe_render_template
+          @handles << ::ActiveSupport::Notifications.subscribe('render_template.action_view') do |*args|
+            handle_render_event(args.last)
+          end
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def subscribe_render_partial
+          @handles << ::ActiveSupport::Notifications.subscribe('render_partial.action_view') do |*args|
+            handle_render_event(args.last)
+          end
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def subscribe_render_collection
+          @handles << ::ActiveSupport::Notifications.subscribe('render_collection.action_view') do |*args|
+            handle_render_event(args.last)
+          end
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def subscribe_sql_active_record
+          @handles << ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
+            handle_sql_event(args.last)
+          end
+        end
+
+        # Rails 7.1 added render_collection.action_view. Subscribing to
+        # a non-existent event silently no-ops on modern AS, but the
+        # version check keeps `@handles` tight for uninstall accounting.
+        def render_collection_supported?
+          return false unless defined?(::Rails::VERSION::STRING)
+
+          Gem::Version.new(::Rails::VERSION::STRING) >= Gem::Version.new('7.1')
+        rescue StandardError
+          false
+        end
+
+        # Pre-digest the schema paths at install time. An unreadable or
+        # missing file is dropped on the floor - graceful degradation.
+        # Returns a frozen array so record_ar_schema can iterate without
+        # defensive dup.
+        def build_schema_inputs(paths)
+          Array(paths).each_with_object([]) do |path, acc|
+            input = try_build_schema_input(path)
+            acc << input if input
+          end.freeze
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def try_build_schema_input(path)
+          abs = File.expand_path(path.to_s, @root)
+          return nil unless abs.start_with?(@root_prefix)
+          return nil unless File.file?(abs)
+
+          digest = Tracker::FileDigest.compute(abs)
+          return nil if digest.nil?
+
+          Tracker::Input.for_file(
+            path: abs, kind: :notification, digest: digest, root: @root
+          )
+        rescue StandardError
+          nil
+        end
+
+        # Internal method on the tracer pipeline.
+        # @api private
+        def safely_unsubscribe(handle)
+          ::ActiveSupport::Notifications.unsubscribe(handle)
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/testsift/rails/preset.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'set'
+
+require_relative '../configuration'
+
+module TestSift
+  # Internal Rails — see {TestSift} for the user-facing surface.
+  # @api private
+  #
+  # Internal Rails-integration subsystem (Railtie + ActiveSupport
+  # notifications + I18n hooks + the default-glob preset). Activated
+  # when the user's spec_helper requires `testsift/rails`.
+  module Rails
+    # Default Rails glob preset - the Coverage-invisible surface that
+    # testsift cannot auto-observe via its standard Ruby-execution
+    # tracking. Enabled by `track_rails_defaults` in .testsift.
+    #
+    # Covered categories (each toggleable via the `except:` kwarg):
+    #
+    #   :views     - app/views/**/* (ERB / Slim / Haml / JBuilder)
+    #   :helpers   - app/helpers/**/*.rb (tracked as declared so helper
+    #                files re-invalidate even before Zeitwerk loads them)
+    #   :locales   - config/locales/**/*.yml (I18n data files)
+    #   :config    - config/*.yml + config/routes.rb (database.yml,
+    #                cable.yml, storage.yml, routes - the YAML + routes
+    #                dispatcher are all config surface)
+    #   :schema    - db/schema.rb + db/structure.sql
+    #   :factories - spec/factories/**/*.rb + test/factories/**/*.rb
+    #   :fixtures  - spec/fixtures/**/*.{yml,yaml,json} + same under test/
+    #
+    # Intentionally omitted:
+    #   - Gemfile.lock, .ruby-version, .testsift: already tracked as
+    #     whole-suite invalidators (Tracker::WholeSuiteInvalidators).
+    #     Listing them as :declared would cross input-kind categories.
+    #   - config/environments/*.rb, config/app_constants.rb: loaded as
+    #     Ruby at Rails boot, so LoadedFilesTracker.@boot_set captures
+    #     them already as transitive-load inputs.
+    #
+    # Unknown `except:` keys raise Configuration::InvalidUsageError, same
+    # pattern as `storage_backend(:bogus)`.
+    class Preset
+      # Internal constant.
+      # @api private
+      DEFAULTS = {
+        views: %w[app/views/**/*].freeze,
+        helpers: %w[app/helpers/**/*.rb].freeze,
+        locales: %w[config/locales/**/*.yml].freeze,
+        config: %w[config/*.yml config/routes.rb].freeze,
+        schema: %w[db/schema.rb db/structure.sql].freeze,
+        factories: %w[spec/factories/**/*.rb test/factories/**/*.rb].freeze,
+        fixtures: %w[
+          spec/fixtures/**/*.{yml,yaml,json}
+          test/fixtures/**/*.{yml,yaml,json}
+        ].freeze
+      }.freeze
+
+      # Internal constant.
+      # @api private
+      ALLOWED_KEYS = DEFAULTS.keys.to_set.freeze
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.globs(except: [])
+        excluded = normalize_excluded(except)
+        validate_excluded!(excluded)
+
+        DEFAULTS
+          .except(*excluded)
+          .values
+          .flatten
+          .uniq
+          .freeze
+      end
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.normalize_excluded(except)
+        Array(except).compact
+      end
+
+      # Internal helper for the tracer pipeline.
+      # @api private
+      def self.validate_excluded!(excluded)
+        unknown = excluded.reject { |key| ALLOWED_KEYS.include?(key) }
+        return if unknown.empty?
+
+        raise TestSift::Configuration::InvalidUsageError,
+              "unknown track_rails_defaults keys: #{unknown.inspect}; " \
+              "allowed: #{ALLOWED_KEYS.to_a.inspect}"
+      end
+    end
+  end
+end

data/lib/testsift/rails/railtie.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module TestSift
+  # Internal Rails — see {TestSift} for the user-facing surface.
+  # @api private
+  module Rails
+    # Rails lifecycle adapter. Only required by `lib/testsift/rails.rb`
+    # when `defined?(::Rails::Railtie)` is true, so loading this file in
+    # a non-Rails process is not expected and will raise NameError.
+    #
+    # Defines the Railtie class + registers one initializer that logs
+    # a single confirmation line. The actual ActiveSupport::Notifications
+    # subscribers for ActionView template renders, I18n lookups, and
+    # schema/factory/fixture tracking live in `notifications.rb` and
+    # `i18n_tracking.rb` and are wired by `Engine.setup`.
+    class Railtie < ::Rails::Railtie
+      initializer 'testsift.setup' do
+        TestSift.logger.info 'testsift Rails integration loaded'
+      end
+    end
+  end
+end

data/lib/testsift/rails.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Rails integration entry point. Loaded explicitly by users who put
+# `require 'testsift/rails'` in their spec_helper - the gem does not
+# auto-load this file, so pure-Ruby suites pay zero cost.
+#
+# Behavior:
+#   - Always loads TestSift::Rails::Preset so Configuration#track_rails_defaults
+#     works whether or not Rails itself is booted in the current process.
+#   - Loads TestSift::Rails::Railtie iff `::Rails::Railtie` is defined,
+#     so requiring this file when Rails is absent is a clean no-op
+#     (AC1: "Rails absent: requiring testsift/rails noops cleanly").
+
+require_relative 'rails/preset'
+require_relative 'rails/railtie' if defined?(Rails::Railtie)

data/lib/testsift/remote_cache/README.md

@@ -0,0 +1,140 @@
+# Remote cache
+
+Optional cross-machine cache layer. Downloads a prior run's dependency
+graph so a fresh checkout (CI, new contributor) starts warm.
+
+## Responsibilities
+
+- Download the cache for a given ref (commit SHA) into the local cache
+  directory, with schema-version validation.
+- Upload the current local cache under the branch ref.
+- Persist branch refs so history rewrites (`git commit --amend`,
+  `git pull -r`) don't invalidate the next run.
+- Prune old entries per configured retention policy.
+
+## Public protocol
+
+```ruby
+module TestSift::RemoteCache::Backend
+  REQUIRED_METHODS = %i[
+    download upload branch_refs write_branch_refs prune! prune_all!
+  ].freeze
+
+  def self.conforms?(backend); end
+end
+```
+
+The shared-examples contract in `spec/contracts/remote_cache_backend.rb`
+asserts the full behavioral contract on every implementation.
+
+## Shipped backends
+
+All three share the two-tier layout (main + per-PR-branch), the same
+schema_version validator, and the same retention knobs.
+
+- **`S3Backend`** — primary target. Shells out to `aws` / `awslocal`
+  CLI. Cache is one `cache.tar.gz` per ref.
+- **`LocalFsBackend`** — shared directory (NFS, dev cache, CI
+  workspace). Same `cache.tar.gz` archive format as S3 — a LocalFs
+  root can be rsync'd to/from S3 with no transform. Atomic uploads via
+  tmp-write + rename. No locking (unreliable over NFS; last-write-wins
+  on same SHA is correct because archive bytes are deterministic).
+- **`RedisBackend`** — each ref stored as a Redis hash (field-per-file)
+  keyed under `<prefix>:main:<sha>` / `<prefix>:pr:<branch>:<sha>`.
+  `redis` gem is an OPTIONAL dependency — users add it to their own
+  Gemfile. A missing gem logs a clear warning and falls back to a cold
+  run instead of crashing the test suite.
+
+## Layout
+
+### S3 / LocalFS (archive-per-ref)
+
+```
+<root>/main/<sha>/[<test_suite_id>/]cache.tar.gz
+<root>/pr/<branch>/<sha>/[<test_suite_id>/]cache.tar.gz
+<root>/pr/<branch>/branch_refs.json
+```
+
+(`<root>` is `s3://<bucket>/<prefix>` for S3, a local/NFS path for
+LocalFs.) Each `cache.tar.gz` packs `last_run.json` + the `<run_id>/`
+directory (the 15-file local layout documented in
+`USER_FACING_SURFACE.md` §6). Upload = 1 PUT; download = 1 GET. Local
+disk layout is unchanged after unpack — external tooling that walks
+`testsift_cache/` still sees the full 15-file breakdown.
+
+### Redis (hash-per-ref)
+
+```
+<prefix>:main:<sha>[:<test_suite_id>]           -> HASH
+<prefix>:pr:<branch>:<sha>[:<test_suite_id>]    -> HASH
+<prefix>:pr:<branch>:branch_refs                -> STRING (JSON)
+```
+
+Hash fields: `_timestamp` (epoch), `last_run.json`, and one field per
+file in the 15-file layout (`<run_id>/<file>.json`). Keeps Redis-native
+inspection (`HGETALL`, `HKEYS`) usable without extracting an archive.
+
+## Tier routing
+
+Tier is determined from `$GIT_BRANCH` vs `$GIT_DEFAULT_BRANCH`. PR
+builds write to the PR tier; main builds write to the main tier. PR
+downloads try their own tier first, then fall back to main tier for
+the same ref (catches cherry-picks from main).
+
+## User-facing surface
+
+The Rakefile (`lib/testsift/remote_cache/Rakefile`) defines three
+tasks:
+
+- `testsift:remote_cache:download`
+- `testsift:remote_cache:upload`
+- `testsift:remote_cache:prune_all` — maintenance task that walks
+  every PR branch and drops ones idle longer than
+  `cache_retention_pr_branch_ttl`. Intended for a nightly cron.
+
+Users load the Rakefile from their own Rakefile:
+
+```ruby
+spec = Gem::Specification.find_by_name('testsift')
+load "#{spec.gem_dir}/lib/testsift/remote_cache/Rakefile"
+```
+
+Env vars: `GIT_DEFAULT_BRANCH` (required), `GIT_BRANCH` (required for
+download/upload; defaults to `GIT_DEFAULT_BRANCH` for prune_all), and
+`TEST_SUITE_ID` (optional; scopes the cache key when set).
+
+## Configuration
+
+Shortest path — one `remote_cache_uri` call:
+
+```ruby
+TestSift.configure do
+  remote_cache_uri 's3://my-bucket/testsift'
+  # or: remote_cache_uri 'file:///mnt/shared-cache'
+  # or: remote_cache_uri 'redis://redis.internal:6379/0'
+  cache_retention_count 100            # keep newest N on main tier
+  cache_retention_pr_branch_ttl '14 days' # idle-branch cutoff on pr tier
+end
+```
+
+Structured form for cases the URI cannot express:
+
+```ruby
+TestSift.configure do
+  remote_cache_backend :s3,       bucket: 'my-bucket', prefix: 'testsift'
+  # or:
+  remote_cache_backend :local_fs, root: '/mnt/shared-cache'
+  # or:
+  remote_cache_backend :redis,    url: ENV.fetch('REDIS_URL'), prefix: 'testsift'
+end
+```
+
+## Caveats
+
+- **LocalFS over NFS**: cross-node consistency is eventual. A
+  download issued by node B immediately after an upload on node A may
+  miss; retries converge. Not a backend correctness bug.
+- **Redis memory**: hash-per-ref without gzip compression uses ~2-5x
+  more bytes than the S3/LocalFS tar+gzip archive. Fine for realistic
+  cache sizes (a few hundred refs × ~100 KB raw); budget Redis memory
+  accordingly for very large fleets.