testsift 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +22 -0
  3. data/LICENSE +22 -0
  4. data/README.md +572 -0
  5. data/bin/testsift +15 -0
  6. data/lib/testsift/cache/Rakefile +43 -0
  7. data/lib/testsift/cli/blast_radius.rb +318 -0
  8. data/lib/testsift/cli/cache_clear.rb +114 -0
  9. data/lib/testsift/cli/cache_info.rb +106 -0
  10. data/lib/testsift/cli/doctor.rb +307 -0
  11. data/lib/testsift/cli/explain.rb +273 -0
  12. data/lib/testsift/cli/report_open.rb +90 -0
  13. data/lib/testsift/cli/snapshot_helpers.rb +119 -0
  14. data/lib/testsift/cli.rb +219 -0
  15. data/lib/testsift/configuration.rb +1467 -0
  16. data/lib/testsift/defaults.rb +11 -0
  17. data/lib/testsift/engine/snapshot_builder.rb +117 -0
  18. data/lib/testsift/engine.rb +896 -0
  19. data/lib/testsift/example.rb +188 -0
  20. data/lib/testsift/filter.rb +103 -0
  21. data/lib/testsift/line_stub.rb +62 -0
  22. data/lib/testsift/load_config.rb +15 -0
  23. data/lib/testsift/load_default_config.rb +46 -0
  24. data/lib/testsift/load_global_config.rb +33 -0
  25. data/lib/testsift/load_local_config.rb +17 -0
  26. data/lib/testsift/logger.rb +59 -0
  27. data/lib/testsift/rails/README.md +78 -0
  28. data/lib/testsift/rails/i18n_tracking.rb +137 -0
  29. data/lib/testsift/rails/notifications.rb +263 -0
  30. data/lib/testsift/rails/preset.rb +94 -0
  31. data/lib/testsift/rails/railtie.rb +22 -0
  32. data/lib/testsift/rails.rb +15 -0
  33. data/lib/testsift/remote_cache/README.md +140 -0
  34. data/lib/testsift/remote_cache/Rakefile +61 -0
  35. data/lib/testsift/remote_cache/archive.rb +137 -0
  36. data/lib/testsift/remote_cache/backend.rb +73 -0
  37. data/lib/testsift/remote_cache/git_ancestry.rb +241 -0
  38. data/lib/testsift/remote_cache/local_fs_backend.rb +439 -0
  39. data/lib/testsift/remote_cache/redis_backend.rb +554 -0
  40. data/lib/testsift/remote_cache/s3_backend.rb +712 -0
  41. data/lib/testsift/remote_cache/user_tasks.rb +436 -0
  42. data/lib/testsift/remote_cache/validator.rb +50 -0
  43. data/lib/testsift/remote_cache.rb +22 -0
  44. data/lib/testsift/reporters/README.md +103 -0
  45. data/lib/testsift/reporters/base.rb +87 -0
  46. data/lib/testsift/reporters/coverage_json_reporter.rb +338 -0
  47. data/lib/testsift/reporters/html/.gitignore +19 -0
  48. data/lib/testsift/reporters/html/.prettierignore +4 -0
  49. data/lib/testsift/reporters/html/.prettierrc.json +9 -0
  50. data/lib/testsift/reporters/html/README.md +80 -0
  51. data/lib/testsift/reporters/html/dist/assets/index.css +2 -0
  52. data/lib/testsift/reporters/html/dist/assets/index.js +1 -0
  53. data/lib/testsift/reporters/html/dist/index.html +24 -0
  54. data/lib/testsift/reporters/html/eslint.config.js +62 -0
  55. data/lib/testsift/reporters/html/package-lock.json +4910 -0
  56. data/lib/testsift/reporters/html/package.json +29 -0
  57. data/lib/testsift/reporters/html/src/app.jsx +130 -0
  58. data/lib/testsift/reporters/html/src/components/AllExamples.jsx +86 -0
  59. data/lib/testsift/reporters/html/src/components/DuplicateExamples.jsx +68 -0
  60. data/lib/testsift/reporters/html/src/components/ExamplesDependency.jsx +78 -0
  61. data/lib/testsift/reporters/html/src/components/FilesDependency.jsx +72 -0
  62. data/lib/testsift/reporters/html/src/components/FlakyExamples.jsx +42 -0
  63. data/lib/testsift/reporters/html/src/components/ReportTable.jsx +131 -0
  64. data/lib/testsift/reporters/html/src/components/SearchBar.jsx +19 -0
  65. data/lib/testsift/reporters/html/src/index.html +23 -0
  66. data/lib/testsift/reporters/html/src/main.jsx +37 -0
  67. data/lib/testsift/reporters/html/src/styles.css +434 -0
  68. data/lib/testsift/reporters/html/vite.config.js +42 -0
  69. data/lib/testsift/reporters/html_reporter.rb +266 -0
  70. data/lib/testsift/reporters/json_reporter.rb +88 -0
  71. data/lib/testsift/reporters/payload_builder.rb +241 -0
  72. data/lib/testsift/reporters/registry.rb +120 -0
  73. data/lib/testsift/reporters/terminal_reporter.rb +264 -0
  74. data/lib/testsift/rspec/README.md +73 -0
  75. data/lib/testsift/rspec/installation.rb +97 -0
  76. data/lib/testsift/rspec/metadata.rb +96 -0
  77. data/lib/testsift/rspec/parallel_tests.rb +459 -0
  78. data/lib/testsift/rspec/rails_setup.rb +42 -0
  79. data/lib/testsift/rspec/reporter_hook.rb +90 -0
  80. data/lib/testsift/rspec/runner_hook.rb +253 -0
  81. data/lib/testsift/source_file.rb +56 -0
  82. data/lib/testsift/storage/README.md +35 -0
  83. data/lib/testsift/storage/backend.rb +130 -0
  84. data/lib/testsift/storage/json_backend.rb +884 -0
  85. data/lib/testsift/storage/lazy_snapshot.rb +65 -0
  86. data/lib/testsift/storage/schema.rb +50 -0
  87. data/lib/testsift/storage/serializer/json.rb +41 -0
  88. data/lib/testsift/storage/serializer/msgpack.rb +167 -0
  89. data/lib/testsift/storage/snapshot.rb +141 -0
  90. data/lib/testsift/storage/sqlite_backend.rb +709 -0
  91. data/lib/testsift/time_formatter.rb +74 -0
  92. data/lib/testsift/tracker/README.md +36 -0
  93. data/lib/testsift/tracker/coverage_adapter.rb +180 -0
  94. data/lib/testsift/tracker/declared_globs.rb +100 -0
  95. data/lib/testsift/tracker/dependency_graph.rb +134 -0
  96. data/lib/testsift/tracker/env_matcher.rb +127 -0
  97. data/lib/testsift/tracker/env_snapshot.rb +77 -0
  98. data/lib/testsift/tracker/example_registry.rb +153 -0
  99. data/lib/testsift/tracker/file_digest.rb +61 -0
  100. data/lib/testsift/tracker/filter.rb +127 -0
  101. data/lib/testsift/tracker/input.rb +99 -0
  102. data/lib/testsift/tracker/io_hooks/file.rb +55 -0
  103. data/lib/testsift/tracker/io_hooks/io.rb +24 -0
  104. data/lib/testsift/tracker/io_hooks/json.rb +23 -0
  105. data/lib/testsift/tracker/io_hooks/kernel.rb +26 -0
  106. data/lib/testsift/tracker/io_hooks/yaml.rb +38 -0
  107. data/lib/testsift/tracker/io_hooks.rb +195 -0
  108. data/lib/testsift/tracker/loaded_files_tracker.rb +290 -0
  109. data/lib/testsift/tracker/new_file_detector.rb +62 -0
  110. data/lib/testsift/tracker/whole_suite_invalidators.rb +96 -0
  111. data/lib/testsift/version.rb +7 -0
  112. data/lib/testsift.rb +335 -0
  113. metadata +193 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 31bf8573c531e759d915952b24e7bb41035376c497afd158e3ac3383fd2c127f
4
+ data.tar.gz: 1b7a909238a8f86851b1e21f75de2afd59e8845c223b58985895c3035bee5b3b
5
+ SHA512:
6
+ metadata.gz: 2d360ad8da5622cc107ae6e679c910d24a267d347ec6eae2a63f642602c7104222bbb2038f21f5006dcc58bfd1d1100d63559541beb815f2bea1d97e3f1fae8d
7
+ data.tar.gz: 45a17c13e5af2218c9ed88935ebbb99893b59af29a0598793fc2c47d883b5eb875ee376d6a616ac806d87db36c808afe86877d53a8644b4b3c94be27ccb57833
data/CHANGELOG.md ADDED
@@ -0,0 +1,22 @@
1
+ # Changelog
2
+
3
+ All notable changes to TestSift. release-please writes each entry from
4
+ the Conventional Commit messages merged since the previous release.
5
+
6
+ ## 0.0.1 (2026-11-09)
7
+
8
+
9
+ ### Features
10
+
11
+ * ship TestSift as its own gem, released by release-please ([04ced03](https://github.com/salasebas/testsift/commit/04ced03513b6c67379c1c50f06cef834590ccffd))
12
+
13
+
14
+ ### Performance Improvements
15
+
16
+ * cut Ruby recorder costs at scale and fix SQLite red-run persistence ([#32](https://github.com/salasebas/testsift/issues/32)) ([1793288](https://github.com/salasebas/testsift/commit/1793288ddaba25bf18cbf658c6a9ac1de50f3853))
17
+ * identify real Coverage costs before a Rust port ([e225fbf](https://github.com/salasebas/testsift/commit/e225fbf240f9828e922aa48f8fd59632dfeab4b0))
18
+
19
+ ## [Upstream history](docs/upstream/CHANGELOG.md)
20
+
21
+ TestSift started from RSpec Tracer `v2.0.0.rc.1`. Upstream releases up to
22
+ that point are preserved in [docs/upstream/CHANGELOG.md](docs/upstream/CHANGELOG.md).
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2021 Abhimanyu Singh
4
+ Copyright (c) 2026 Sebastian Sala
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in
14
+ all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,572 @@
1
+ # TestSift
2
+
3
+ [![CI](https://github.com/salasebas/testsift/actions/workflows/ci.yml/badge.svg?branch=testsift)](https://github.com/salasebas/testsift/actions/workflows/ci.yml?query=branch%3Atestsift)
4
+ [![codecov](https://codecov.io/gh/salasebas/testsift/branch/testsift/graph/badge.svg)](https://codecov.io/gh/salasebas/testsift)
5
+
6
+ **Affected-test selection for RSpec: re-run only the examples whose
7
+ recorded inputs changed, with flaky-test detection and a per-example
8
+ dependency map.**
9
+
10
+ TestSift is an independent project that started as a fork of
11
+ [RSpec Tracer](https://github.com/avmnu-sng/rspec-tracer) at upstream
12
+ `v2.0.0.rc.1` (commit `f13851f7b3393aaafa7c7df088c1605747f07ce5`). It
13
+ ships its own gem, `testsift`, versioned from `0.0.1`; it is not
14
+ upstream's 2.0 release. [FORK.md](FORK.md) records the baseline and
15
+ attribution, [UPGRADING.md](UPGRADING.md) covers moving from
16
+ `rspec-tracer`, and the
17
+ [TestSift roadmap](https://github.com/salasebas/testsift/issues/1)
18
+ tracks engine extraction, coverage integration, and a Minitest adapter.
19
+
20
+ No TestSift version has been published to RubyGems yet; until one is,
21
+ install from this repository.
22
+
23
+ TestSift records the inputs it can observe for each RSpec
24
+ example -- Ruby files, file I/O, Rails templates and queries,
25
+ declared globs and ENV branches -- and turns that record into a
26
+ flaky-test detector, a per-example dependency map, and optional CI
27
+ acceleration. The flaky report and the dependency map never skip a
28
+ test: value before trust.
29
+
30
+ **Measured, not promised.** In the upstream maintainer's May 2026
31
+ field test (rspec-tracer 2.0.0.pre.1, Apple M2 Max), Mastodon's `spec/lib` --
32
+ 1,234 examples, Rails 8.1.3, Ruby 3.4.8 -- ran in 116.4 s cold and
33
+ 17.0 s warm: **6.85x**, with the tracer's own recording overhead
34
+ included in both runs. "Cold" here is the tracer's own first run,
35
+ not plain RSpec: recording costs roughly 1.6x plain RSpec on this
36
+ repository's Rails benchmark fixture (approximate), and no
37
+ tracer-free Mastodon baseline was measured, so your gain over a
38
+ suite without the tracer will be smaller than cold minus warm.
39
+ Suite shape matters: across the same field tests, warm-run skip
40
+ rates ranged from 31% (a suite carrying 177 pre-existing failures --
41
+ failed examples are never skipped) to 100%. Measure your own suite
42
+ before you rely on these numbers.
43
+
44
+ **Conservative by design, honest about its limits.** The tracer
45
+ never skips failed, flaky, or pending examples, nor any example
46
+ whose recorded inputs changed; when a recorded input is ambiguous,
47
+ it re-runs. Real blind spots exist -- runtime metaprogramming and
48
+ monkey-patches are invisible to it; the `tracks:` DSL is the escape
49
+ hatch -- and the [soundness model](ARCHITECTURE.md#soundness-model)
50
+ catalogues the known blind spots. Shadow mode (run everything,
51
+ report what would have been skipped) is planned in
52
+ [#9](https://github.com/salasebas/testsift/issues/9), so you can
53
+ verify the tracer on your own data before trusting it with a single
54
+ skip.
55
+
56
+ **Your data never leaves your infrastructure.** Everything runs inside
57
+ your test process and your CI cache -- no SaaS backend, no telemetry,
58
+ no per-seat pricing.
59
+
60
+ ## Quick start
61
+
62
+ Requires Ruby 3.1+ and rspec-core 3.12+ (Rails 7.0+ when using Rails;
63
+ Rails 8.0 needs Ruby 3.2+; JRuby 9.4 supported). Every CI-gated Ruby
64
+ is supported until at least six months past its upstream EOL -- see
65
+ [Maintenance](#maintenance).
66
+
67
+ ```ruby
68
+ # Gemfile -- 0.x releases may change the public API; pin the minor
69
+ gem 'testsift', github: 'salasebas/testsift', branch: 'testsift',
70
+ group: :test, require: false
71
+ # After the first RubyGems release:
72
+ # gem 'testsift', '~> 0.0.1', group: :test, require: false
73
+ ```
74
+
75
+ Add the tracer's working directories to your `.gitignore` before the
76
+ first run:
77
+
78
+ ```
79
+ testsift.lock
80
+ testsift_cache/
81
+ testsift_coverage/
82
+ testsift_report/
83
+ ```
84
+
85
+ ```ruby
86
+ # Top of spec_helper.rb / rails_helper.rb, before any application
87
+ # code. With SimpleCov, start SimpleCov first -- load order is part
88
+ # of the contract.
89
+ require 'testsift'
90
+ TestSift.start
91
+ ```
92
+
93
+ Run the suite twice. The second run skips the examples whose recorded
94
+ inputs are unchanged and prints a per-reason breakdown of what
95
+ re-ran; open `testsift_report/index.html` to audit every
96
+ per-example decision. Not ready to let it skip anything? Set
97
+ `TESTSIFT_RUN_ALL_EXAMPLES=true` (or `run_all_examples true`
98
+ inside the `TestSift.configure` block in `.testsift`) --
99
+ every example still runs while the flaky report and the dependency
100
+ maps accumulate data.
101
+
102
+ Setting up CI? [docs/CI_RECIPES.md](docs/CI_RECIPES.md) has
103
+ copy-paste cache recipes for CircleCI, GitLab CI, Buildkite, and
104
+ Heroku CI, plus a link to the canonical GitHub Actions workflow.
105
+
106
+ ## Table of contents
107
+
108
+ - [Quick start](#quick-start)
109
+ - [What it saves](#what-it-saves)
110
+ - [How it works](#how-it-works)
111
+ - [Per-example `tracks:` DSL](#per-example-tracks-dsl)
112
+ - [Rails quick start](#rails-quick-start)
113
+ - [Configuring CI](#configuring-ci)
114
+ - [Remote cache backends](#remote-cache-backends)
115
+ - [Pluggable storage](#pluggable-storage)
116
+ - [Command-line tools](#command-line-tools)
117
+ - [SimpleCov interop](#simplecov-interop)
118
+ - [FAQ + comparison](#faq--comparison)
119
+ - [Reports](#reports)
120
+ - [Maintenance](#maintenance)
121
+ - [Documentation + coverage](#documentation--coverage)
122
+ - [Help and community](#help-and-community)
123
+
124
+ ## What it saves
125
+
126
+ All timings are wall-clock and measured, not projected: upstream
127
+ maintainer field tests, May 2026, Apple M2 Max, rspec-tracer
128
+ 2.0.0.pre.1, before the TestSift rename. In
129
+ every row, "cold" is the tracer's own first, recording run -- it
130
+ costs more than plain RSpec (roughly 1.6x on this repository's Rails
131
+ benchmark fixture; approximate), and no tracer-free baseline was
132
+ measured for these projects -- so your savings versus a suite
133
+ without the tracer will be smaller than cold minus warm.
134
+
135
+ | Project measured | Examples | Cold run | Warm run | Warm skip rate | Saved per warm run |
136
+ |---|---|---|---|---|---|
137
+ | Mastodon `spec/lib` (Rails 8.1.3, Ruby 3.4.8) | 1,234 | 116.4 s | 17.0 s | 98% (1,212 skipped) | ~99 s |
138
+ | thoughtbot/clearance (Rails 8 engine + dummy app) | 258 | 11.1 s | 1.6 s | 100% | ~9.5 s |
139
+ | publify (carries 177 pre-existing failures; failed examples are never skipped) | 257 | 6.7 s | 4.8 s | 31% | ~1.9 s |
140
+
141
+ The Mastodon warm run is the zero-change best case; in the same
142
+ field test, editing one file re-ran 68 of 1,234 examples in 19.0 s.
143
+ Laptop rows understate CI savings: CI runners are slower than the
144
+ development machine, so the seconds saved per run are typically
145
+ larger there -- the skip ratios are what transfer.
146
+
147
+ **Worked examples** (every input is an assumption you should replace
148
+ with your own). For any suite:
149
+
150
+ `(cold seconds - warm seconds) x runs/day / 60 x $/CI-minute x 30 = $/month per full-suite job`
151
+
152
+ At $0.006/CI-minute (illustrative -- GitHub-hosted Linux list price
153
+ at the time of writing; check your provider's current rate) and 50
154
+ CI runs/day on one job:
155
+
156
+ - Best case (a Mastodon-sized suite saving ~99 s per warm run):
157
+ ~83 CI-minutes/day, about $15/month per job.
158
+ - Low-skip case (a publify-shaped suite saving ~1.9 s per warm run
159
+ at 31% skip): about $0.29/month per job.
160
+
161
+ Multiply by the number of jobs that each run the full suite (for
162
+ example, a Ruby-version matrix); jobs that split one suite across
163
+ workers share the saving rather than multiply it.
164
+
165
+ Conservative assumptions: 50 runs/day and the dollar rate are stated
166
+ assumptions, not measurements; your skip rate may be far lower (see
167
+ the 31% row -- suites with many failing examples skip less, by
168
+ design); remote-cache download time on fresh CI workers is not
169
+ subtracted (measured end-to-end remote-cache gain on clearance was
170
+ still ~4.6x).
171
+
172
+ ## How it works
173
+
174
+ Each example's observed inputs -- executed Ruby source (via
175
+ `Coverage`), file I/O (via `Module#prepend` hooks), Rails framework
176
+ events, declared globs and env vars, and whole-suite invalidators
177
+ like `Gemfile.lock` -- are digested and stored per example. The next
178
+ run recomputes the digests and skips examples whose recorded inputs
179
+ are unchanged. The full input taxonomy lives in
180
+ [ARCHITECTURE.md](ARCHITECTURE.md#input-taxonomy); what each input
181
+ kind does and does not guarantee is the
182
+ [soundness model](ARCHITECTURE.md#soundness-model); the Rails
183
+ `config.eager_load` precision trade-off is covered by the Rails
184
+ recipe in [COOKBOOK.md](COOKBOOK.md).
185
+
186
+ ## Per-example `tracks:` DSL
187
+
188
+ Annotate any describe / context / example with extra inputs the
189
+ tracker can't auto-observe — config files baked at boot, env-var
190
+ branches, non-Ruby deps:
191
+
192
+ ```ruby
193
+ RSpec.describe AdminController,
194
+ tracks: { files: 'app/policies/**/*.rb',
195
+ env: 'ROLE_CONFIG' } do
196
+ it 'gates on the feature flag' do
197
+ # ...
198
+ end
199
+ end
200
+ ```
201
+
202
+ | Key | Value shape | Semantics |
203
+ |----------|-----------------------------------|------------------------------------------------------------------------|
204
+ | `:files` | String glob OR Array of strings | Each matched file is attached as a `:declared`-kind dep on the example. |
205
+ | `:env` | String name OR Array OR wildcard | Each named env var is digested at finalize. Missing env reads as empty. |
206
+
207
+ **Wildcards** (`'RAILS_*'`, `'*_TOKEN'`, `'*'`) expand against the
208
+ live ENV at register time; the persisted snapshot only carries
209
+ concrete keys.
210
+
211
+ **Cascade.** Nested groups contribute additively. A parent's
212
+ `tracks: { files: 'a/*' }` and a child's `tracks: { env: 'B' }` both
213
+ attach to the child's examples; the child does not clobber the parent
214
+ on any shared key.
215
+
216
+ **Use `track_files` / `track_env` for the global case.** Files /
217
+ env vars **every** test depends on (`Gemfile.lock`, `AUTH_TOKEN`,
218
+ `RAILS_ENV`) belong in `.testsift`:
219
+
220
+ ```ruby
221
+ TestSift.configure do
222
+ track_files 'config/locales/**/*.yml', 'db/schema.rb', 'Gemfile.lock'
223
+ track_env 'AUTH_TOKEN', 'DATABASE_URL', 'RAILS_*'
224
+ end
225
+ ```
226
+
227
+ ## Rails quick start
228
+
229
+ ```ruby
230
+ # spec/rails_helper.rb
231
+ require 'simplecov' # if used; load BEFORE testsift
232
+ SimpleCov.start
233
+
234
+ require 'testsift'
235
+ TestSift.start
236
+
237
+ require File.expand_path('../config/environment', __dir__)
238
+ require 'rspec/rails'
239
+ ```
240
+
241
+ ```ruby
242
+ # .testsift
243
+ TestSift.configure do
244
+ track_rails_defaults
245
+ end
246
+ ```
247
+
248
+ `track_rails_defaults` attaches the common Rails-side declared globs:
249
+ views, locales, fixtures, factories, helpers, config. Drop a
250
+ specific glob to hand attribution off to the per-example subscribers
251
+ instead:
252
+
253
+ ```ruby
254
+ TestSift.configure do
255
+ # Templates → render_template.action_view subscriber attributes
256
+ # them per-example. Schema → opt-in sql.active_record observer
257
+ # attributes db/schema.rb + db/structure.sql per AR-touching
258
+ # example (read the Narrow AR-schema attribution section before
259
+ # enabling).
260
+ track_rails_defaults except: [:views, :schema]
261
+ track_ar_schema_notifications
262
+ end
263
+ ```
264
+
265
+ Supported Rails matrix: 7.0 / 7.1 / 7.2 / 8.0 (8.0 requires Ruby
266
+ 3.2+). On JRuby use `JRUBY_OPTS="--debug -X+O"` and the JDBC
267
+ adapter (`gem 'activerecord-jdbcsqlite3-adapter', '~> 71.0',
268
+ platforms: :jruby` for Rails 7.1; track the major to your Rails
269
+ line).
270
+
271
+ ### Narrow AR-schema attribution
272
+
273
+ `track_ar_schema_notifications` promises per-example attribution of
274
+ `db/schema.rb` via the `sql.active_record` subscriber. **The narrow
275
+ promise only holds when no per-example AR cleanup mechanism fires
276
+ queries inside the per-example bucket.** Common Rails setups trip
277
+ this:
278
+
279
+ - `use_transactional_fixtures = true` (Rails default) — per-example
280
+ BEGIN/COMMIT fires `sql.active_record`.
281
+ - DatabaseCleaner `:truncation` / `:deletion` / `:transaction` in
282
+ `around` hooks — cleanup queries fire inside the bucket.
283
+
284
+ Either case attributes `db/schema.rb` to every AR-touching example
285
+ (safe, but widens invalidation). A boot-time warn fires when the
286
+ precondition is unmet so you don't discover this from a confused
287
+ cache-hit-rate chart later.
288
+
289
+ For genuinely narrow attribution: set `use_transactional_fixtures =
290
+ false` and use sequence-based factories (or another non-AR cleanup
291
+ mechanism that doesn't fire `sql.active_record` inside the example
292
+ window).
293
+
294
+ ### Working with parallel_tests
295
+
296
+ Supported out of the box. The tracker writes to per-worker
297
+ directories and merges at finalize on the elected last worker. If you
298
+ interrupt a run mid-flight, delete the lock file:
299
+
300
+ ```sh
301
+ rm -f testsift.lock && bundle exec parallel_rspec
302
+ ```
303
+
304
+ ## Configuring CI
305
+
306
+ Copy-paste cache recipes for CircleCI, GitLab CI, Buildkite, and
307
+ Heroku CI live in [docs/CI_RECIPES.md](docs/CI_RECIPES.md); the
308
+ canonical GitHub Actions workflow is
309
+ [`.github/workflows/example-tracer-cache.yml`](.github/workflows/example-tracer-cache.yml).
310
+ The rest of this section covers the remote-cache rake flow and the
311
+ cache-key pattern all the recipes share.
312
+
313
+ The rspec-tracer 1.x flow is preserved bit-for-bit. In your project's `Rakefile`:
314
+
315
+ ```ruby
316
+ spec = Gem::Specification.find_by_name('testsift')
317
+ load "#{spec.gem_dir}/lib/testsift/remote_cache/Rakefile"
318
+ ```
319
+
320
+ Then in your CI pipeline:
321
+
322
+ ```sh
323
+ bundle exec rake testsift:remote_cache:download
324
+ bundle exec rspec
325
+ bundle exec rake testsift:remote_cache:upload
326
+ ```
327
+
328
+ With these env vars:
329
+
330
+ - **`GIT_DEFAULT_BRANCH`** — your repo's default branch (`main` /
331
+ `master`).
332
+ - **`GIT_BRANCH`** — the branch the build is on.
333
+ - **`TEST_SUITES`** — total number of suite shards (when sharding).
334
+ - **`TEST_SUITE_ID`** — current shard identifier.
335
+ - **`TESTSIFT_REMOTE_CACHE_URI`** — the remote-cache URI
336
+ (`s3://bucket/prefix`, `file:///tmp/cache`, `redis://host`).
337
+
338
+ ### Caching TestSift in CI
339
+
340
+ If you don't have S3 — or only need cache between runs of a single
341
+ workflow — GitHub Actions' built-in cache is a drop-in storage
342
+ substrate. Restore `testsift_cache/` before specs, run the
343
+ suite, let the post-step save it back. A reference workflow lives at
344
+ [`.github/workflows/example-tracer-cache.yml`](.github/workflows/example-tracer-cache.yml).
345
+ The cache step:
346
+
347
+ ```yaml
348
+ - name: Restore testsift cache
349
+ uses: actions/cache@v5
350
+ with:
351
+ path: testsift_cache
352
+ key: >-
353
+ ${{ runner.os
354
+ }}-${{ hashFiles('.ruby-version')
355
+ }}-${{ hashFiles('Gemfile.lock')
356
+ }}-testsift-cache
357
+ restore-keys: |
358
+ ${{ runner.os }}-${{ hashFiles('.ruby-version') }}-
359
+ ${{ runner.os }}-
360
+ ```
361
+
362
+ The 4-component cache key (`runner.os` + Ruby version + your
363
+ project's Gemfile lock -- which also pins the TestSift gem's
364
+ version -- + a fixed name suffix) invalidates when something would
365
+ make the previous run's decisions incorrect: native gem binaries
366
+ differ across runner OSes, Ruby ABI
367
+ changes invalidate native extensions, a tracer upgrade can change the
368
+ cache schema, and gem-set drift is the most common cache-staleness
369
+ trigger.
370
+
371
+ **The pattern translates 1:1 to CircleCI, GitLab CI, Buildkite, and
372
+ Heroku CI**; only the YAML envelope is GHA-specific. See
373
+ [`docs/CI_RECIPES.md`](docs/CI_RECIPES.md) for per-provider recipes.
374
+
375
+ ## Remote cache backends
376
+
377
+ Three backends ship. Pick one in `.testsift`:
378
+
379
+ ```ruby
380
+ TestSift.configure do
381
+ # S3 (preserves the rspec-tracer 1.x layout; supports awslocal / LocalStack).
382
+ remote_cache_backend :s3, local: false
383
+
384
+ # Filesystem-backed (no S3 needed).
385
+ # remote_cache_backend :local_fs, root: '/tmp/testsift-cache'
386
+
387
+ # Redis (with optional per-key TTL + PR-branch tracking sidecar).
388
+ # remote_cache_backend :redis, url: ENV['REDIS_URL'], ttl: 7 * 86_400
389
+ end
390
+ ```
391
+
392
+ The `rake testsift:remote_cache:*` task surface is unchanged —
393
+ backend selection happens in config; the rake tasks dispatch
394
+ identically.
395
+
396
+ ## Pluggable storage
397
+
398
+ Two on-disk storage formats:
399
+
400
+ ```ruby
401
+ TestSift.configure do
402
+ # JSON (default) — preserves the rspec-tracer 1.x 10-file layout per run.
403
+ storage_backend :json
404
+
405
+ # SQLite — single-file database. Faster cold reads above ~5,000
406
+ # examples. JRuby auto-falls-back to :json with a one-time warn.
407
+ # storage_backend :sqlite
408
+ end
409
+ ```
410
+
411
+ Or override per-run via env: `TESTSIFT_STORAGE=sqlite`.
412
+
413
+ ## Command-line tools
414
+
415
+ `testsift` exposes six sub-commands. Run them via Bundler so the
416
+ gem's executable resolves cleanly without needing `bundle binstubs
417
+ testsift` first:
418
+
419
+ ```sh
420
+ bundle exec testsift doctor # diagnose config + environment
421
+ bundle exec testsift cache:info # size, last run, invalidation stats
422
+ bundle exec testsift cache:clear # rm cache dirs
423
+ bundle exec testsift report:open # open the HTML report
424
+ bundle exec testsift explain <id> # why is <example_id> scheduled to (re-)run?
425
+ bundle exec testsift blast-radius <file> # which examples re-run if <file> changes?
426
+ ```
427
+
428
+ Generated binstubs (`bin/testsift …`) work too once you've run
429
+ `bundle binstubs testsift` in your project. The CLI is opt-in for
430
+ local-dev convenience; the `rake testsift:remote_cache:*` tasks
431
+ remain first-class for CI integration — nothing in the CLI replaces
432
+ them.
433
+
434
+ ## SimpleCov interop
435
+
436
+ **Load order is part of the contract.** SimpleCov first, then
437
+ TestSift:
438
+
439
+ ```ruby
440
+ require 'simplecov'
441
+ SimpleCov.start
442
+
443
+ require 'testsift'
444
+ TestSift.start
445
+ ```
446
+
447
+ If you call `require 'simplecov'` but skip `SimpleCov.start` before
448
+ `TestSift.start`, a boot-time warn fires pointing this out
449
+ (silent-degradation breaks coverage output).
450
+
451
+ **Branch coverage works alongside TestSift.** The rspec-tracer 1.x
452
+ caveat ("SimpleCov would not report branch coverage results even
453
+ when enabled") is no longer applicable — the coverage emission
454
+ decoupled from SimpleCov's branch-tracking. Re-enable
455
+ `enable_coverage :branch` in your `SimpleCov.start` block and you get
456
+ both.
457
+
458
+ Filter chains compose: SimpleCov's `add_filter` / `coverage_filters`
459
+ control SimpleCov's HTML output; TestSift's `add_filter` /
460
+ `add_coverage_filter` control which files contribute to the
461
+ dependency graph.
462
+
463
+ ## FAQ + comparison
464
+
465
+ **Why not just SimpleCov filtering?** SimpleCov tells you which
466
+ *files* are covered. TestSift tells you which *examples* depend
467
+ on which inputs, so it can skip the ones whose inputs didn't change.
468
+ The two solve adjacent problems; you typically want both.
469
+
470
+ **Knapsack / Knapsack Pro / Test Boosters?** Those are test-splitting
471
+ tools — they shard your suite across CI workers. TestSift is
472
+ orthogonal: it skips already-passing examples on a single worker.
473
+ Compose them: shard with Knapsack, then skip with TestSift.
474
+ A composition smoke spec (`spec/regressions/knapsack_coexistence_spec.rb`)
475
+ proves they coexist.
476
+
477
+ **Datadog Test Optimization / Buildkite Test Analytics / CircleCI
478
+ Test Insights?** Those are hosted services: your test metadata ships
479
+ to a vendor backend, with the pricing and procurement that implies.
480
+ TestSift runs entirely inside your test process -- the cache and
481
+ reports are plain files in your repo, S3 bucket, or Redis. No SaaS
482
+ backend, no telemetry, no per-seat pricing, and TestSift keeps it
483
+ that way.
484
+
485
+ **`RSpec::Retry` / `RSpec::Rerun`?** Retries flaky failures.
486
+ TestSift detects flaky examples (same inputs, different outcomes)
487
+ and refuses to skip them on the next run. Compose: retry to make a
488
+ flaky suite green, let TestSift flag the flakes for you to
489
+ actually fix.
490
+
491
+ **Monorepo with N apps?** Set per-app `cache_dir` in each app's
492
+ `.testsift` (the default `testsift_cache/` would otherwise
493
+ collide across apps).
494
+
495
+ **`tracks:` vs `track_rails_defaults` overlap?** When both attribute
496
+ the same file to an example, the declared-glob attribution wins.
497
+ Deterministic; see [`ARCHITECTURE.md`](ARCHITECTURE.md) "Input
498
+ taxonomy" for the rule.
499
+
500
+ ## Reports
501
+
502
+ After the run, `testsift_report/index.html` opens five HTML
503
+ reports. The first two deliver value even if you never let the
504
+ tracer skip a single example:
505
+
506
+ - **Flaky Examples** -- examples that failed on one run and passed on
507
+ a later one, the canonical intermittence signal: a flake list your
508
+ suite builds passively from the runs you were already doing. Once
509
+ flagged, the tracer refuses to skip them on subsequent runs.
510
+ - **Files Dependency** -- "if I change this file, which tests run?"
511
+ The file-to-test map that makes refactors, reviews, and deletions
512
+ safer -- the report unique to TestSift.
513
+ - **Examples Dependency** -- the per-example inverse: "what does this
514
+ test depend on?"
515
+ - **All Examples** -- basic test info (id, status, duration, the
516
+ inputs the example consumed).
517
+ - **Duplicate Examples** -- pairs RSpec couldn't uniquely identify
518
+ (file:line collisions; only appears when duplicates are present).
519
+
520
+ Plus a machine-readable `testsift_report/report.json` for CI
521
+ dashboards, and a terminal summary with a per-reason breakdown after
522
+ every run (see [Quick start](#quick-start)).
523
+
524
+ ## Maintenance
525
+
526
+ Every CI-gated Ruby is supported until **at least** six months past
527
+ its upstream end-of-life date. That is a floor, not a drop date:
528
+ versions can stay supported longer, and Ruby 3.1 currently is.
529
+ Upstream EOL dates are published on the official
530
+ [Ruby maintenance schedule](https://www.ruby-lang.org/en/downloads/branches/);
531
+ that page is authoritative for the dates below.
532
+
533
+ | Version | CI-gated | Status |
534
+ |---------|----------|--------|
535
+ | Ruby 3.1 | Yes | Supported. Upstream EOL was 2025-03-26, so the committed six-month window has lapsed -- and 3.1 is still supported today because the policy is a minimum, not a schedule. It remains the floor. |
536
+ | Ruby 3.2 | Yes | Supported. Upstream EOL was 2026-04-01; supported until at least 2026-10-01, and potentially longer -- the policy is a minimum. |
537
+ | Ruby 3.3 | Yes | Supported. Upstream EOL is expected 2027-03-31; supported until at least six months past the published EOL date. |
538
+ | Ruby 3.4 | Yes | Supported, until at least six months past its upstream EOL date (not yet announced). |
539
+ | Ruby 4.0 | Yes | Supported, until at least six months past its upstream EOL date (not yet announced). |
540
+ | JRuby 9.4 | Yes | Supported. JRuby support tracks the JRuby project's own maintenance schedule, not the Ruby-Core EOL dates above. |
541
+
542
+ TruffleRuby and Ruby head are best-effort: no CI gate, but issue
543
+ reports are welcome and get investigated.
544
+
545
+ ## Documentation + coverage
546
+
547
+ - **API reference**: `task docs:yard` builds the YARD reference
548
+ locally. A hosted docs site is not published for TestSift yet.
549
+
550
+ - **Coverage**: tracked by [Codecov](https://codecov.io/gh/salasebas/testsift)
551
+ (per-PR delta + diff-coverage gate ≥ 90%; project history). Multi-
552
+ suite resultsets (`unit`, `edge-cases`, `regressions-plain`, `fuzz`)
553
+ merge via SimpleCov.collate in the CI `coverage` job; the same
554
+ merger runs locally via `task coverage:merge` after `task test:*`
555
+ runs (open `coverage/index.html` for the local HTML report).
556
+
557
+ ## Help and community
558
+
559
+ - **Bug reports + feature requests** — [GitHub Issues](https://github.com/salasebas/testsift/issues).
560
+ Include Ruby / Rails / RSpec / SimpleCov versions and your
561
+ `.testsift` config.
562
+ - **Roadmap** — [TestSift roadmap](https://github.com/salasebas/testsift/issues/1).
563
+ - **Architecture deep-dive** — [`ARCHITECTURE.md`](ARCHITECTURE.md).
564
+ - **Moving from rspec-tracer** — [`UPGRADING.md`](UPGRADING.md).
565
+ - **Upstream history** — [`docs/upstream/`](docs/upstream/) keeps the
566
+ RSpec Tracer changelog, roadmap, and 1.x → 2.0 upgrade guide as
567
+ they were at the fork point.
568
+ - **Contributing** — [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md).
569
+
570
+ Released under the [MIT License](LICENSE). Everyone interacting in
571
+ the project is expected to follow the
572
+ [Code of Conduct](.github/CODE_OF_CONDUCT.md).
data/bin/testsift ADDED
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ # Entry point for the `testsift` CLI. The binary is opt-in for
5
+ # users who want diagnostics, cache inspection, or report-opening
6
+ # from the command line; the canonical CI flow continues to go
7
+ # through `rake testsift:remote_cache:*` tasks per
8
+ # USER_FACING_SURFACE.md §5.
9
+
10
+ lib_dir = File.expand_path('../lib', __dir__)
11
+ $LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir)
12
+
13
+ require 'testsift/cli'
14
+
15
+ exit TestSift::CLI.run(ARGV)
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ # User-facing Rake task shim for local-cache maintenance. Load from
4
+ # the user's own Rakefile when they want one-off cleanup:
5
+ #
6
+ # spec = Gem::Specification.find_by_name('testsift')
7
+ # load "#{spec.gem_dir}/lib/testsift/cache/Rakefile"
8
+ #
9
+ # Defines `testsift:cache:gc` which prunes old run-id
10
+ # directories under the local cache while retaining the `N`
11
+ # most-recent configured via `cache_retention_local_count` (default
12
+ # 5). Prune-on-save already handles the steady-state cleanup; this
13
+ # task is for catching up after retention was opted-out-of and the
14
+ # cache grew beyond the configured cap.
15
+
16
+ require 'testsift'
17
+ require 'testsift/storage/json_backend'
18
+
19
+ namespace :testsift do
20
+ namespace :cache do
21
+ desc 'Prune old run-id directories under the local cache ' \
22
+ '(respects cache_retention_local_count)'
23
+ task :gc do
24
+ count = TestSift.cache_retention_local_count
25
+ if count.nil? || count.zero?
26
+ TestSift.logger.warn(
27
+ 'testsift cache gc: cache_retention_local_count is 0 (opt-out); nothing to prune'
28
+ )
29
+ next
30
+ end
31
+
32
+ backend = TestSift::Storage::JsonBackend.new(
33
+ cache_path: TestSift.cache_path,
34
+ logger: TestSift.logger,
35
+ retention_local_count: count
36
+ )
37
+ removed = backend.prune_run_dirs!(keep: count)
38
+ TestSift.logger.info(
39
+ "testsift cache gc: pruned #{removed} run-id dir(s); kept #{count} most-recent"
40
+ )
41
+ end
42
+ end
43
+ end