backspin 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +3 -0
  3. data/.gitignore +1 -0
  4. data/.ruby-version +1 -1
  5. data/.standard.yml +5 -0
  6. data/CHANGELOG.md +13 -4
  7. data/Gemfile.lock +27 -25
  8. data/README.md +80 -1
  9. data/Rakefile +27 -0
  10. data/backspin.gemspec +3 -1
  11. data/docs/compare-api-plan.md +193 -0
  12. data/fixtures/projects/dummy_cli_gem/.rspec +3 -0
  13. data/fixtures/projects/dummy_cli_gem/Gemfile +12 -0
  14. data/fixtures/projects/dummy_cli_gem/Gemfile.lock +55 -0
  15. data/fixtures/projects/dummy_cli_gem/LICENSE.txt +1 -0
  16. data/fixtures/projects/dummy_cli_gem/README.md +7 -0
  17. data/fixtures/projects/dummy_cli_gem/Rakefile +8 -0
  18. data/fixtures/projects/dummy_cli_gem/dummy_cli_gem.gemspec +22 -0
  19. data/fixtures/projects/dummy_cli_gem/exe/dummy_cli_gem +8 -0
  20. data/fixtures/projects/dummy_cli_gem/fixtures/backspin/dummy_echo.yml +18 -0
  21. data/fixtures/projects/dummy_cli_gem/fixtures/backspin/dummy_ls.yml +18 -0
  22. data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem/cli.rb +35 -0
  23. data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem/version.rb +5 -0
  24. data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem.rb +7 -0
  25. data/fixtures/projects/dummy_cli_gem/mise.toml +2 -0
  26. data/fixtures/projects/dummy_cli_gem/spec/dummy_cli_gem_backspin_spec.rb +46 -0
  27. data/fixtures/projects/dummy_cli_gem/spec/fixtures/backspin/dummy_echo.yml +18 -0
  28. data/fixtures/projects/dummy_cli_gem/spec/fixtures/backspin/dummy_ls.yml +18 -0
  29. data/fixtures/projects/dummy_cli_gem/spec/fixtures/listing_target/alpha.txt +1 -0
  30. data/fixtures/projects/dummy_cli_gem/spec/spec_helper.rb +22 -0
  31. data/lib/backspin/configuration.rb +13 -0
  32. data/lib/backspin/record.rb +1 -1
  33. data/lib/backspin/recorder.rb +1 -1
  34. data/lib/backspin/version.rb +1 -1
  35. data/lib/backspin.rb +91 -25
  36. data/mise.toml +2 -0
  37. metadata +39 -5
  38. data/script/run_affected_tests +0 -179
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a9533baf54aa3ae09b9d483d41149a6ede985f3f974f038e287034928a0ab1e3
4
- data.tar.gz: 75b9cb2018b1f1ca5878f6ce75af74bc9fc4822319ac2f12e8c654f30624f859
3
+ metadata.gz: 3810a17b9b213b010cadd7419559fda50aed8dee3faad5865e824e765921339d
4
+ data.tar.gz: 400594d38ca7937da9095c78eb6578d5d384077a1cbbf78db97f06e133c6dcd0
5
5
  SHA512:
6
- metadata.gz: a8d851c2836d8cd0be15b341a7f2af332c356aaa9688766a5b51f65110539a2d500cac8238abd4bc0a668a689effbef45a8f009829a3cd28ff4acd249bb506b4
7
- data.tar.gz: 0151e1de9a47285ca75552e97154b57100de50fe691100869144e1ed17e1b7c421a012599c4462a333f6abd0c4779f110a4fbffd2bbe60d14d600469da459bad
6
+ metadata.gz: c2614110086b787c10b31504ae0eed80b32adc7a72d61709f6fba6245e093b35f92816227faebf781b3257e25b766c45676c47e74422176ae07fbc3eafaabce2
7
+ data.tar.gz: 6bb11f527a1aebc8ed495ccde7ca4deddf87b33f33f160cf76da13e02bc994e9b397cbaa59df2f30656cbd4a589ea70ebe24194743ac96a9b639fcda4cd9bf22
data/.circleci/config.yml CHANGED
@@ -20,6 +20,9 @@ jobs:
20
20
  - run:
21
21
  name: Run specs
22
22
  command: bundle exec rake spec
23
+ - run:
24
+ name: Run fake gem specs
25
+ command: bundle exec rake spec:fake_gem
23
26
 
24
27
  lint:
25
28
  resource_class: medium
data/.gitignore CHANGED
@@ -7,6 +7,7 @@
7
7
  /tmp/
8
8
  /.claude
9
9
  /.cursor
10
+ /checksums/
10
11
 
11
12
  /backspin*.gem
12
13
  # rspec failure tracking
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 4.0.0
1
+ 4
data/.standard.yml CHANGED
@@ -2,6 +2,11 @@
2
2
  parallel: true
3
3
  format: progress
4
4
 
5
+ # Match the gemspec's required_ruby_version. Without this, Standard infers the
6
+ # target from .ruby-version (4) and flags requires that are only redundant on
7
+ # Ruby >= 3.5 - removing them would break the 3.x versions we still support.
8
+ ruby_version: 3.2
9
+
5
10
  ignore:
6
11
  - 'tmp/**/*'
7
12
  - 'vendor/**/*'
data/CHANGELOG.md CHANGED
@@ -1,12 +1,22 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.13.0
4
+ * Added `Backspin.compare(reference:, actual:)` for differential testing - runs both commands live and compares their filtered output, with no record file. Only stdout, stderr, and exit status are compared, so the two commands may differ in argv and env without normalization.
5
+ * Added `Backspin::ReferenceCommandError`, raised when a compare's reference command produces no output at all (usually a sign it failed to start), rather than comparing empty to empty.
6
+ * Raised the minimum supported Ruby to 3.2 (3.1 reached end-of-life in March 2025).
7
+ * Fixed snapshot timestamps to be recorded as UTC rather than local time, so re-recording in a different timezone no longer produces noisy diffs.
8
+ * Fixed a missing `require "time"`, which could raise `NoMethodError` on `Time#iso8601` in consumers that never load the time stdlib themselves.
9
+
10
+ ## 0.12.0 - 2026-02-11
11
+ * Added `BACKSPIN_MODE` environment variable to globally override recording mode (`auto`, `record`, `verify`).
12
+ * Explicit `mode:` kwarg still takes highest precedence, followed by the env var, then auto-detection.
13
+ * Added configurable logger to `Backspin::Configuration` (defaults to WARN level, logfmt-lite format, and can be disabled with `config.logger = nil`).
14
+
3
15
  ## 0.11.0 - 2026-02-11
4
16
  * Added immutable top-level `first_recorded_at` metadata for record files.
5
17
  * Added mutable top-level `recorded_at` metadata that updates on each successful re-record.
6
18
  * Added top-level `record_count`, incremented on each successful record write.
7
- * Record format now writes `format_version: 4.1`; loading remains backward-compatible with 4.0 record files.
8
- * Added acceptance coverage for v4.1 schema and 4.0-to-4.1 upgrade behavior.
9
- * Removed legacy committed `.yml` record fixtures from old schema versions.
19
+ * Bumped record format to 4.1; loading remains backward-compatible with 4.0 record files.
10
20
 
11
21
  ## 0.10.0 - 2026-02-11
12
22
  * Added `filter_on` to `Backspin.run` and `Backspin.capture` (`:both` default, `:record` opt-out).
@@ -19,7 +29,6 @@
19
29
  * Breaking: result convenience accessors (`result.stdout`, `result.stderr`, `result.status`) were removed in favor of snapshot access.
20
30
  * Breaking: record format bumped to 4.0 and now persists a single `snapshot` object (v3 records are rejected).
21
31
  * Simplification: removed legacy `Command`, `CommandResult`, and `RecordResult` layers; matcher/diff now operate directly on snapshots.
22
- * Added focused coverage for the new result contract and capture stream restoration behavior.
23
32
  * Updated project docs to reflect the BackspinResult + Snapshot API surface.
24
33
 
25
34
  ## 0.8.0 - 2026-02-05
data/Gemfile.lock CHANGED
@@ -1,70 +1,72 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- backspin (0.11.0)
4
+ backspin (0.13.0)
5
+ logger
5
6
 
6
7
  GEM
7
8
  remote: https://rubygems.org/
8
9
  specs:
9
10
  ast (2.4.3)
10
11
  diff-lcs (1.6.2)
11
- json (2.16.0)
12
- language_server-protocol (3.17.0.5)
12
+ json (2.21.1)
13
+ language_server-protocol (3.17.0.6)
13
14
  lint_roller (1.1.0)
14
- parallel (1.27.0)
15
- parser (3.3.10.0)
15
+ logger (1.7.0)
16
+ parallel (1.28.0)
17
+ parser (3.3.12.0)
16
18
  ast (~> 2.4.1)
17
19
  racc
18
- prism (1.6.0)
20
+ prism (1.9.0)
19
21
  racc (1.8.1)
20
22
  rainbow (3.1.1)
21
- rake (13.3.0)
22
- regexp_parser (2.11.3)
23
- rspec (3.13.1)
23
+ rake (13.4.2)
24
+ regexp_parser (2.12.0)
25
+ rspec (3.13.2)
24
26
  rspec-core (~> 3.13.0)
25
27
  rspec-expectations (~> 3.13.0)
26
28
  rspec-mocks (~> 3.13.0)
27
- rspec-core (3.13.4)
29
+ rspec-core (3.13.6)
28
30
  rspec-support (~> 3.13.0)
29
31
  rspec-expectations (3.13.5)
30
32
  diff-lcs (>= 1.2.0, < 2.0)
31
33
  rspec-support (~> 3.13.0)
32
- rspec-mocks (3.13.5)
34
+ rspec-mocks (3.13.8)
33
35
  diff-lcs (>= 1.2.0, < 2.0)
34
36
  rspec-support (~> 3.13.0)
35
- rspec-support (3.13.4)
36
- rubocop (1.81.7)
37
+ rspec-support (3.13.7)
38
+ rubocop (1.88.2)
37
39
  json (~> 2.3)
38
40
  language_server-protocol (~> 3.17.0.2)
39
41
  lint_roller (~> 1.1.0)
40
- parallel (~> 1.10)
42
+ parallel (>= 1.10)
41
43
  parser (>= 3.3.0.2)
42
44
  rainbow (>= 2.2.2, < 4.0)
43
45
  regexp_parser (>= 2.9.3, < 3.0)
44
- rubocop-ast (>= 1.47.1, < 2.0)
46
+ rubocop-ast (>= 1.49.0, < 2.0)
45
47
  ruby-progressbar (~> 1.7)
46
48
  unicode-display_width (>= 2.4.0, < 4.0)
47
- rubocop-ast (1.48.0)
49
+ rubocop-ast (1.50.0)
48
50
  parser (>= 3.3.7.2)
49
- prism (~> 1.4)
50
- rubocop-performance (1.25.0)
51
+ prism (~> 1.7)
52
+ rubocop-performance (1.26.1)
51
53
  lint_roller (~> 1.1)
52
54
  rubocop (>= 1.75.0, < 2.0)
53
- rubocop-ast (>= 1.38.0, < 2.0)
55
+ rubocop-ast (>= 1.47.1, < 2.0)
54
56
  ruby-progressbar (1.13.0)
55
- standard (1.52.0)
57
+ standard (1.56.0)
56
58
  language_server-protocol (~> 3.17.0.2)
57
59
  lint_roller (~> 1.0)
58
- rubocop (~> 1.81.7)
60
+ rubocop (~> 1.88.0)
59
61
  standard-custom (~> 1.0.0)
60
62
  standard-performance (~> 1.8)
61
63
  standard-custom (1.0.2)
62
64
  lint_roller (~> 1.0)
63
65
  rubocop (~> 1.50)
64
- standard-performance (1.8.0)
66
+ standard-performance (1.9.0)
65
67
  lint_roller (~> 1.1)
66
- rubocop-performance (~> 1.25.0)
67
- timecop (0.9.10)
68
+ rubocop-performance (~> 1.26.0)
69
+ timecop (0.9.11)
68
70
  unicode-display_width (3.2.0)
69
71
  unicode-emoji (~> 4.1)
70
72
  unicode-emoji (4.2.0)
@@ -81,4 +83,4 @@ DEPENDENCIES
81
83
  timecop (~> 0.9)
82
84
 
83
85
  BUNDLED WITH
84
- 4.0.5
86
+ 4.0.16
data/README.md CHANGED
@@ -75,6 +75,25 @@ end
75
75
 
76
76
  Block capture records a single combined stdout/stderr snapshot. Exit status is a placeholder (`0`) in this mode.
77
77
 
78
+ ### Differential Testing (Compare)
79
+
80
+ Use `Backspin.compare` when the question is "does my tool produce the same output as this reference tool?" Both commands run live and their output is compared directly - nothing is recorded to disk:
81
+
82
+ ```ruby
83
+ result = Backspin.compare(
84
+ reference: ["bundle", "exec", "rspec", "spec/failing_spec.rb"],
85
+ actual: ["my-runner", "spec/failing_spec.rb"]
86
+ )
87
+
88
+ result.expected.stdout # reference output
89
+ result.actual.stdout # output from the command under test
90
+ result.verified? # true when they match
91
+ ```
92
+
93
+ Only stdout, stderr, and exit status are compared, so the two commands can differ in argv and environment without any normalization. `filter` and `matcher` work exactly as they do for `Backspin.run`, and a mismatch raises `Backspin::VerificationError` unless `raise_on_verification_failure` is disabled.
94
+
95
+ If the reference command produces no output at all - usually a sign it failed to start rather than that its output is genuinely empty - `compare` raises `Backspin::ReferenceCommandError` instead of comparing empty to empty.
96
+
78
97
  ### Recording Modes
79
98
 
80
99
  Backspin supports different modes for controlling how commands are recorded and verified:
@@ -91,6 +110,26 @@ result = Backspin.run(["echo", "hello"], name: "echo_test", mode: :verify)
91
110
  expect(result.verified?).to be true
92
111
  ```
93
112
 
113
+ ### Environment Variable Mode Override
114
+
115
+ Set `BACKSPIN_MODE` to globally force a recording mode without changing any test code:
116
+
117
+ ```bash
118
+ # Re-record all fixtures
119
+ BACKSPIN_MODE=record bundle exec rspec
120
+
121
+ # Verify-only (CI, no accidental re-records)
122
+ BACKSPIN_MODE=verify bundle exec rspec
123
+ ```
124
+
125
+ Precedence (highest to lowest):
126
+
127
+ 1. Explicit `mode:` kwarg (`:record` or `:verify`)
128
+ 2. `BACKSPIN_MODE` environment variable
129
+ 3. Auto-detection (record if no file exists, verify if it does)
130
+
131
+ Allowed values: `auto`, `record`, `verify` (case-insensitive). Invalid values raise `ArgumentError`.
132
+
94
133
  ### Record Metadata
95
134
 
96
135
  Backspin writes records using `format_version: "4.1"` with top-level metadata:
@@ -191,7 +230,7 @@ Backspin.run(["echo", "id=123"], name: "record_only", filter: normalize_filter,
191
230
 
192
231
  ### Working with the Result Object
193
232
 
194
- The API returns a `Backspin::BackspinResult` object with helpful methods:
233
+ The API returns a `Backspin::BackspinResult` object with details on run.
195
234
 
196
235
  ```ruby
197
236
  result = Backspin.run(["sh", "-c", "echo out; echo err >&2; exit 42"], name: "my_test")
@@ -243,6 +282,34 @@ result = Backspin.run(["echo", "different"], name: "my_test")
243
282
  Backspin.reset_configuration!
244
283
  ```
245
284
 
285
+ ### Logging
286
+
287
+ Backspin includes a configurable logger for diagnostics. By default it is set to WARN level; but most messages are logged at DEBUG level.
288
+ So if you are looking for more detailed logs, you can set the logger to DEBUG level:
289
+
290
+ ```ruby
291
+ Backspin.configure do |config|
292
+ config.logger = Logger.new($stdout)
293
+ config.logger.level = Logger::DEBUG
294
+ end
295
+ ```
296
+
297
+ To replace the logger entirely:
298
+
299
+ ```ruby
300
+ Backspin.configure do |config|
301
+ config.logger = Logger.new("log/backspin.log")
302
+ end
303
+ ```
304
+
305
+ To disable Backspin logging entirely (for example in tests):
306
+
307
+ ```ruby
308
+ Backspin.configure do |config|
309
+ config.logger = nil
310
+ end
311
+ ```
312
+
246
313
  ### Credential Scrubbing
247
314
 
248
315
  If the CLI interaction you are recording contains sensitive data in stdout/stderr, you should be careful to make sure it is not recorded to YAML.
@@ -274,6 +341,18 @@ Automatic scrubbing includes:
274
341
 
275
342
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests.
276
343
 
344
+ This repo also includes a decoupled full-stack fixture gem at `fixtures/projects/dummy_cli_gem` that uses Backspin the way downstream projects do. Run it with:
345
+
346
+ ```bash
347
+ bundle exec rake full_stack:dummy_app
348
+ ```
349
+
350
+ To re-record that fixture's committed YAML snapshots:
351
+
352
+ ```bash
353
+ bundle exec rake full_stack:record_dummy_app
354
+ ```
355
+
277
356
  To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
278
357
 
279
358
  ## Contributing
data/Rakefile CHANGED
@@ -6,6 +6,33 @@ require "standard/rake"
6
6
 
7
7
  RSpec::Core::RakeTask.new(:spec)
8
8
 
9
+ def run_in_fake_gem(command)
10
+ dummy_app_dir = File.expand_path("fixtures/projects/dummy_cli_gem", __dir__)
11
+
12
+ Bundler.with_unbundled_env do
13
+ Dir.chdir(dummy_app_dir) do
14
+ sh "bundle check || bundle install"
15
+ sh command
16
+ end
17
+ end
18
+ end
19
+
20
+ namespace :spec do
21
+ desc "Run the dummy fixture gem specs with Backspin (decoupled from main suite)"
22
+ task :fake_gem do
23
+ run_in_fake_gem("bundle exec rspec")
24
+ end
25
+
26
+ desc "Re-record Backspin YAML fixtures for the dummy fixture gem"
27
+ task :fake_gem_record do
28
+ original_record_mode = ENV["RECORD_MODE"]
29
+ ENV["RECORD_MODE"] = "record"
30
+ run_in_fake_gem("bundle exec rspec")
31
+ ensure
32
+ ENV["RECORD_MODE"] = original_record_mode
33
+ end
34
+ end
35
+
9
36
  task default: %i[spec standard]
10
37
 
11
38
  load "release.rake" if File.exist?("release.rake")
data/backspin.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
12
12
  spec.description = "Backspin is a Ruby library for characterization testing of command-line interfaces. Inspired by VCR's cassette-based approach, it records and replays CLI interactions to make testing faster and more deterministic."
13
13
  spec.homepage = "https://github.com/rsanheim/backspin"
14
14
  spec.license = "MIT"
15
- spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
16
16
 
17
17
  spec.metadata["homepage_uri"] = spec.homepage
18
18
  spec.metadata["source_code_uri"] = spec.homepage
@@ -24,4 +24,6 @@ Gem::Specification.new do |spec|
24
24
  spec.bindir = "exe"
25
25
  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
26
26
  spec.require_paths = ["lib"]
27
+
28
+ spec.add_dependency "logger"
27
29
  end
@@ -0,0 +1,193 @@
1
+ # Compare API Plan — differential testing between two commands
2
+
3
+ Date: 2026-07-20 (revised 2026-07-25)
4
+
5
+ ## Motivation
6
+
7
+ The most common way Backspin is used in the `plur` test suite is *differential
8
+ testing*: "does tool B produce the same output as reference tool A?" plur is a
9
+ Go reimplementation of a parallel RSpec runner, and its golden specs assert that
10
+ plur's output matches `rspec`'s own output byte-for-byte.
11
+
12
+ Backspin has no first-class API for this, so today it is expressed as an
13
+ implicit two-call dance with the same record name:
14
+
15
+ ```ruby
16
+ # record rspec baseline
17
+ chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
18
+ # verify plur against it
19
+ result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
20
+ expect(result.verified?).to be(true)
21
+ ```
22
+
23
+ That pattern works but has sharp edges, each of which has bitten a real plur
24
+ spec:
25
+
26
+ 1. **Ordering is load-bearing and invisible.** The first `run` records only
27
+ because the record file happens not to exist yet; if it does exist, *both*
28
+ calls verify against the stored baseline — a different meaning than the code
29
+ reads like.
30
+ 2. **A stale baseline requires a manual `rm`.** Change the fixture and the
31
+ committed `.yml` no longer matches the reference command, so the first call
32
+ fails on `reference-vs-stale-baseline`. Nothing signals that deleting the
33
+ snapshot is the fix.
34
+ 3. **The command line must be neutralized by hand.** The two commands *differ*
35
+ on purpose (`rspec …` vs `plur …`), so specs stuff `"args" => ["[PLACEHOLDER]"]`
36
+ into the filter to stop the recorded command from mismatching.
37
+ 4. **The filter is passed twice** and must be identical on both calls.
38
+ 5. **The dance can span two examples.** plur's minitest golden test records in
39
+ one `it` and verifies in a *different* `it` — order-dependent across
40
+ examples, so it only works because RSpec runs in defined order.
41
+
42
+ The gap is small and specific: `Backspin.run` assumes the *record command* and
43
+ the *verify command* are the same. Differential testing needs them to differ.
44
+
45
+ ## The idea
46
+
47
+ Add `Backspin.compare`, which runs a **reference** command and an **actual**
48
+ command and compares their (filtered) output. It is pure orchestration over
49
+ primitives Backspin already has — no new filtering, snapshot, diff, matcher, or
50
+ result code, and no record file.
51
+
52
+ ```ruby
53
+ Backspin.compare(
54
+ reference:, # String/Array command that defines correct output
55
+ actual:, # String/Array command under test
56
+ env: nil,
57
+ matcher: nil, # reused as-is
58
+ filter: nil # reused as-is
59
+ )
60
+ # => BackspinResult
61
+ ```
62
+
63
+ `reference` maps to `result.expected`, `actual` maps to `result.actual` — which
64
+ is exactly the existing `BackspinResult` contract (expected = baseline, actual =
65
+ what just ran). See `docs/backspin-result-api-sketch.md`.
66
+
67
+ ## Scope: live-vs-live only
68
+
69
+ An earlier draft also proposed a snapshot-backed mode (`name:` + `mode:`), where
70
+ `compare` would record the reference to a `.yml` and later verify `actual`
71
+ against it. That is deliberately **not** in scope:
72
+
73
+ - In `mode: :verify` the `reference:` argument would be silently ignored. You
74
+ could change the reference command and nothing would happen until someone
75
+ deleted the record — sharp edge #2 wearing a new hat, plus a dead keyword arg.
76
+ - It roughly doubles the API surface (`name`, `mode`, `BACKSPIN_MODE`,
77
+ `filter_on`) for a caller that does not exist yet. plur always has `rspec`
78
+ available, in dev and in CI.
79
+
80
+ If a real caller shows up with a reference command that is expensive or absent
81
+ in CI, add it then. `Backspin.run` already covers "freeze one command's output."
82
+
83
+ ## Reuse map — what each need already maps to
84
+
85
+ | Need | Existing primitive | Location |
86
+ |---|---|---|
87
+ | Run a command → stdout/stderr/status | `execute_command(command, env)` | `lib/backspin.rb` |
88
+ | Build a snapshot value object | `Snapshot.new(command_type: Open3::Capture3, …)` | `lib/backspin/snapshot.rb` |
89
+ | Apply `filter` to both sides | `CommandDiff#build_comparison_snapshot` | `lib/backspin/command_diff.rb` |
90
+ | Compare (default = stdout/stderr/status; custom matcher/hash) | `CommandDiff` + `Matcher` | `lib/backspin/command_diff.rb`, `lib/backspin/matcher.rb` |
91
+ | `verified?` / `diff` / `summary` | `CommandDiff` | `lib/backspin/command_diff.rb` |
92
+ | Result object (`expected`, `actual`, `verified?`, `diff`, `error_message`) | `BackspinResult` | `lib/backspin/backspin_result.rb` |
93
+ | Raise on mismatch (config-gated) | `raise_on_verification_failure!(result)` | `lib/backspin.rb` |
94
+
95
+ The heart of it: `CommandDiff.new(expected:, actual:, matcher:, filter:)`
96
+ already takes two snapshots, applies the filter to both, runs the matcher, and
97
+ produces `verified?`/`diff`. `compare` just has to hand it two snapshots.
98
+
99
+ ## Semantics that fall out for free
100
+
101
+ - **args/env are not compared.** The default matcher only looks at stdout,
102
+ stderr, status (`Matcher#evaluate_default`), and `CommandDiff#diff` only diffs
103
+ those three. So two intentionally-different commands compare cleanly with no
104
+ `args` placeholder — deleting plur boilerplate.
105
+ - **command_type matches.** Both sides are `Open3::Capture3`, so
106
+ `CommandDiff#command_types_match?` is true.
107
+ - **Strict-by-default is preserved.** `raise_on_verification_failure!` already
108
+ raises `VerificationError` (with `result.diff`) unless
109
+ `raise_on_verification_failure = false`. `compare` reuses it verbatim.
110
+
111
+ ## Guard: a broken reference must not pass
112
+
113
+ Live-vs-live has one failure mode the two-call dance did not: if the reference
114
+ command fails to *run at all* — wrong directory, `bundle` not on `PATH`, a load
115
+ error — it produces empty stdout and a non-zero status. If `actual` fails the
116
+ same way, `verified?` is true and the spec goes green while testing nothing.
117
+
118
+ `compare` raises `Backspin::ReferenceCommandError` when the reference command
119
+ produces no output on either stream. Exit status can't be the signal: plur's
120
+ golden specs deliberately run failing suites that exit 1.
121
+
122
+ ## The new code
123
+
124
+ 1. `Backspin.compare` public method (validation + orchestration + the guard).
125
+ 2. A private `capture_command_snapshot(command, env)` wrapping
126
+ `execute_command` + `Snapshot.new`. This is **extracted from existing
127
+ duplication** — `perform_command_run` builds that same snapshot twice, so
128
+ pulling it out DRYs `run` as well.
129
+ 3. One fix in `raise_on_verification_failure!`: only emit the `Record:` line
130
+ when there is a record path, so `compare` failures don't render a dangling
131
+ `Record: ` label.
132
+
133
+ No changes to `CommandDiff`, `Matcher`, `Snapshot`, `Record`, or
134
+ `BackspinResult`. `compare` lives beside `run`/`capture` in `lib/backspin.rb`.
135
+
136
+ ### Notes
137
+
138
+ - There is no `filter_on:` keyword. `filter_on: :record` would mean "filter only
139
+ when persisting", and `compare` never persists — passing it would silently
140
+ disable filtering. Omitting the keyword makes that a plain `ArgumentError`.
141
+ - `record_path` is `nil` on the result, and `mode` is `:verify` (nothing was
142
+ recorded, so `recorded?` is false).
143
+
144
+ ## Before / after (plur golden spec)
145
+
146
+ Today (`spec/integration/spec/aggregate_failure_golden_spec.rb` in plur):
147
+
148
+ ```ruby
149
+ chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
150
+ result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
151
+ expect(result.verified?).to be(true)
152
+ ```
153
+
154
+ With `compare`:
155
+
156
+ ```ruby
157
+ result = chdir(fixture) do
158
+ Backspin.compare(reference: rspec_cmd, actual: plur_cmd, filter: normalize)
159
+ end
160
+ expect(result.verified?).to be(true)
161
+ ```
162
+
163
+ A `dir:` option to drop the `chdir` wrapper is a possible follow-up; not needed
164
+ here, since one wrapper now covers what took two.
165
+
166
+ ## Success criteria
167
+
168
+ 1. `Backspin.compare(reference:, actual:)` runs both and returns a
169
+ `BackspinResult` with `expected` = reference snapshot, `actual` = actual
170
+ snapshot, and boolean `verified?`.
171
+ 2. On mismatch, `result.diff` / `result.error_message` are populated by the
172
+ existing `CommandDiff`, and `VerificationError` is raised unless
173
+ `raise_on_verification_failure = false`. The failure message has no empty
174
+ `Record:` line.
175
+ 3. `filter` / `matcher` behave exactly as in `run` (same objects, same code
176
+ paths).
177
+ 4. Two commands with different argv compare on output only — no `args`
178
+ normalization needed.
179
+ 5. No record file is written or read.
180
+ 6. A reference command that produces no output raises
181
+ `ReferenceCommandError` rather than comparing empty-to-empty.
182
+ 7. No new filtering / snapshot / diff / matcher / result implementation — only a
183
+ `compare` entry point, the extracted `capture_command_snapshot` helper, and
184
+ the nil-`record_path` message fix.
185
+
186
+ ## Downstream: what this buys plur
187
+
188
+ Six call-site pairs across four files collapse to a single call each:
189
+ `single_failure_golden_spec.rb` (three examples), `aggregate_failure_golden_spec.rb`,
190
+ `pending_output_spec.rb`, and `minitest_integration_spec.rb` (the cross-example
191
+ one). Five fixture YAMLs get deleted, along with every `"args" => [...]`
192
+ placeholder line. plur's other five Backspin specs are ordinary single-command
193
+ snapshot tests and are untouched.
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gemspec
6
+
7
+ gem "backspin", path: "../../.."
8
+
9
+ group :development, :test do
10
+ gem "rake", "~> 13"
11
+ gem "rspec", "~> 3"
12
+ end
@@ -0,0 +1,55 @@
1
+ PATH
2
+ remote: ../../..
3
+ specs:
4
+ backspin (0.13.0)
5
+ logger
6
+
7
+ PATH
8
+ remote: .
9
+ specs:
10
+ dummy_cli_gem (0.1.0)
11
+
12
+ GEM
13
+ remote: https://rubygems.org/
14
+ specs:
15
+ diff-lcs (1.6.2)
16
+ logger (1.7.0)
17
+ rake (13.3.1)
18
+ rspec (3.13.2)
19
+ rspec-core (~> 3.13.0)
20
+ rspec-expectations (~> 3.13.0)
21
+ rspec-mocks (~> 3.13.0)
22
+ rspec-core (3.13.6)
23
+ rspec-support (~> 3.13.0)
24
+ rspec-expectations (3.13.5)
25
+ diff-lcs (>= 1.2.0, < 2.0)
26
+ rspec-support (~> 3.13.0)
27
+ rspec-mocks (3.13.7)
28
+ diff-lcs (>= 1.2.0, < 2.0)
29
+ rspec-support (~> 3.13.0)
30
+ rspec-support (3.13.7)
31
+
32
+ PLATFORMS
33
+ aarch64-linux
34
+ ruby
35
+
36
+ DEPENDENCIES
37
+ backspin!
38
+ dummy_cli_gem!
39
+ rake (~> 13)
40
+ rspec (~> 3)
41
+
42
+ CHECKSUMS
43
+ backspin (0.13.0)
44
+ diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
45
+ dummy_cli_gem (0.1.0)
46
+ logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
47
+ rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
48
+ rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
49
+ rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
50
+ rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
51
+ rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c
52
+ rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
53
+
54
+ BUNDLED WITH
55
+ 4.0.5
@@ -0,0 +1 @@
1
+ MIT License
@@ -0,0 +1,7 @@
1
+ # DummyCliGem Fixture
2
+
3
+ This is a minimal fixture gem used by Backspin's full-stack integration checks.
4
+
5
+ - It shells out to standard unix utilities (`echo`, `ls`).
6
+ - Its RSpec suite verifies command output through `Backspin.run`.
7
+ - Snapshot YAML records are stored in-repo under `spec/fixtures/backspin`.
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "lib/dummy_cli_gem/version"
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "dummy_cli_gem"
7
+ spec.version = DummyCliGem::VERSION
8
+ spec.authors = ["Backspin"]
9
+ spec.email = ["noreply@example.com"]
10
+ spec.summary = "Dummy CLI gem fixture for Backspin full-stack verification"
11
+ spec.description = "Fixture gem that shells out to unix utilities and is tested via Backspin snapshots."
12
+ spec.homepage = "https://example.com/dummy_cli_gem"
13
+ spec.license = "MIT"
14
+ spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
15
+
16
+ spec.files = Dir.chdir(__dir__) do
17
+ Dir["lib/**/*.rb", "exe/*", "spec/**/*", "script/**/*", "README.md", "LICENSE.txt"]
18
+ end
19
+ spec.bindir = "exe"
20
+ spec.executables = ["dummy_cli_gem"]
21
+ spec.require_paths = ["lib"]
22
+ end