backspin 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +3 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.standard.yml +5 -0
- data/CHANGELOG.md +13 -4
- data/Gemfile.lock +27 -25
- data/README.md +80 -1
- data/Rakefile +27 -0
- data/backspin.gemspec +3 -1
- data/docs/compare-api-plan.md +193 -0
- data/fixtures/projects/dummy_cli_gem/.rspec +3 -0
- data/fixtures/projects/dummy_cli_gem/Gemfile +12 -0
- data/fixtures/projects/dummy_cli_gem/Gemfile.lock +55 -0
- data/fixtures/projects/dummy_cli_gem/LICENSE.txt +1 -0
- data/fixtures/projects/dummy_cli_gem/README.md +7 -0
- data/fixtures/projects/dummy_cli_gem/Rakefile +8 -0
- data/fixtures/projects/dummy_cli_gem/dummy_cli_gem.gemspec +22 -0
- data/fixtures/projects/dummy_cli_gem/exe/dummy_cli_gem +8 -0
- data/fixtures/projects/dummy_cli_gem/fixtures/backspin/dummy_echo.yml +18 -0
- data/fixtures/projects/dummy_cli_gem/fixtures/backspin/dummy_ls.yml +18 -0
- data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem/cli.rb +35 -0
- data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem/version.rb +5 -0
- data/fixtures/projects/dummy_cli_gem/lib/dummy_cli_gem.rb +7 -0
- data/fixtures/projects/dummy_cli_gem/mise.toml +2 -0
- data/fixtures/projects/dummy_cli_gem/spec/dummy_cli_gem_backspin_spec.rb +46 -0
- data/fixtures/projects/dummy_cli_gem/spec/fixtures/backspin/dummy_echo.yml +18 -0
- data/fixtures/projects/dummy_cli_gem/spec/fixtures/backspin/dummy_ls.yml +18 -0
- data/fixtures/projects/dummy_cli_gem/spec/fixtures/listing_target/alpha.txt +1 -0
- data/fixtures/projects/dummy_cli_gem/spec/spec_helper.rb +22 -0
- data/lib/backspin/configuration.rb +13 -0
- data/lib/backspin/record.rb +1 -1
- data/lib/backspin/recorder.rb +1 -1
- data/lib/backspin/version.rb +1 -1
- data/lib/backspin.rb +91 -25
- data/mise.toml +2 -0
- metadata +39 -5
- data/script/run_affected_tests +0 -179
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3810a17b9b213b010cadd7419559fda50aed8dee3faad5865e824e765921339d
|
|
4
|
+
data.tar.gz: 400594d38ca7937da9095c78eb6578d5d384077a1cbbf78db97f06e133c6dcd0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c2614110086b787c10b31504ae0eed80b32adc7a72d61709f6fba6245e093b35f92816227faebf781b3257e25b766c45676c47e74422176ae07fbc3eafaabce2
|
|
7
|
+
data.tar.gz: 6bb11f527a1aebc8ed495ccde7ca4deddf87b33f33f160cf76da13e02bc994e9b397cbaa59df2f30656cbd4a589ea70ebe24194743ac96a9b639fcda4cd9bf22
|
data/.circleci/config.yml
CHANGED
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4
|
|
1
|
+
4
|
data/.standard.yml
CHANGED
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
parallel: true
|
|
3
3
|
format: progress
|
|
4
4
|
|
|
5
|
+
# Match the gemspec's required_ruby_version. Without this, Standard infers the
|
|
6
|
+
# target from .ruby-version (4) and flags requires that are only redundant on
|
|
7
|
+
# Ruby >= 3.5 - removing them would break the 3.x versions we still support.
|
|
8
|
+
ruby_version: 3.2
|
|
9
|
+
|
|
5
10
|
ignore:
|
|
6
11
|
- 'tmp/**/*'
|
|
7
12
|
- 'vendor/**/*'
|
data/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.13.0
|
|
4
|
+
* Added `Backspin.compare(reference:, actual:)` for differential testing - runs both commands live and compares their filtered output, with no record file. Only stdout, stderr, and exit status are compared, so the two commands may differ in argv and env without normalization.
|
|
5
|
+
* Added `Backspin::ReferenceCommandError`, raised when a compare's reference command produces no output at all (usually a sign it failed to start), rather than comparing empty to empty.
|
|
6
|
+
* Raised the minimum supported Ruby to 3.2 (3.1 reached end-of-life in March 2025).
|
|
7
|
+
* Fixed snapshot timestamps to be recorded as UTC rather than local time, so re-recording in a different timezone no longer produces noisy diffs.
|
|
8
|
+
* Fixed a missing `require "time"`, which could raise `NoMethodError` on `Time#iso8601` in consumers that never load the time stdlib themselves.
|
|
9
|
+
|
|
10
|
+
## 0.12.0 - 2026-02-11
|
|
11
|
+
* Added `BACKSPIN_MODE` environment variable to globally override recording mode (`auto`, `record`, `verify`).
|
|
12
|
+
* Explicit `mode:` kwarg still takes highest precedence, followed by the env var, then auto-detection.
|
|
13
|
+
* Added configurable logger to `Backspin::Configuration` (defaults to WARN level, logfmt-lite format, and can be disabled with `config.logger = nil`).
|
|
14
|
+
|
|
3
15
|
## 0.11.0 - 2026-02-11
|
|
4
16
|
* Added immutable top-level `first_recorded_at` metadata for record files.
|
|
5
17
|
* Added mutable top-level `recorded_at` metadata that updates on each successful re-record.
|
|
6
18
|
* Added top-level `record_count`, incremented on each successful record write.
|
|
7
|
-
*
|
|
8
|
-
* Added acceptance coverage for v4.1 schema and 4.0-to-4.1 upgrade behavior.
|
|
9
|
-
* Removed legacy committed `.yml` record fixtures from old schema versions.
|
|
19
|
+
* Bumped record format to 4.1; loading remains backward-compatible with 4.0 record files.
|
|
10
20
|
|
|
11
21
|
## 0.10.0 - 2026-02-11
|
|
12
22
|
* Added `filter_on` to `Backspin.run` and `Backspin.capture` (`:both` default, `:record` opt-out).
|
|
@@ -19,7 +29,6 @@
|
|
|
19
29
|
* Breaking: result convenience accessors (`result.stdout`, `result.stderr`, `result.status`) were removed in favor of snapshot access.
|
|
20
30
|
* Breaking: record format bumped to 4.0 and now persists a single `snapshot` object (v3 records are rejected).
|
|
21
31
|
* Simplification: removed legacy `Command`, `CommandResult`, and `RecordResult` layers; matcher/diff now operate directly on snapshots.
|
|
22
|
-
* Added focused coverage for the new result contract and capture stream restoration behavior.
|
|
23
32
|
* Updated project docs to reflect the BackspinResult + Snapshot API surface.
|
|
24
33
|
|
|
25
34
|
## 0.8.0 - 2026-02-05
|
data/Gemfile.lock
CHANGED
|
@@ -1,70 +1,72 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
backspin (0.
|
|
4
|
+
backspin (0.13.0)
|
|
5
|
+
logger
|
|
5
6
|
|
|
6
7
|
GEM
|
|
7
8
|
remote: https://rubygems.org/
|
|
8
9
|
specs:
|
|
9
10
|
ast (2.4.3)
|
|
10
11
|
diff-lcs (1.6.2)
|
|
11
|
-
json (2.
|
|
12
|
-
language_server-protocol (3.17.0.
|
|
12
|
+
json (2.21.1)
|
|
13
|
+
language_server-protocol (3.17.0.6)
|
|
13
14
|
lint_roller (1.1.0)
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
logger (1.7.0)
|
|
16
|
+
parallel (1.28.0)
|
|
17
|
+
parser (3.3.12.0)
|
|
16
18
|
ast (~> 2.4.1)
|
|
17
19
|
racc
|
|
18
|
-
prism (1.
|
|
20
|
+
prism (1.9.0)
|
|
19
21
|
racc (1.8.1)
|
|
20
22
|
rainbow (3.1.1)
|
|
21
|
-
rake (13.
|
|
22
|
-
regexp_parser (2.
|
|
23
|
-
rspec (3.13.
|
|
23
|
+
rake (13.4.2)
|
|
24
|
+
regexp_parser (2.12.0)
|
|
25
|
+
rspec (3.13.2)
|
|
24
26
|
rspec-core (~> 3.13.0)
|
|
25
27
|
rspec-expectations (~> 3.13.0)
|
|
26
28
|
rspec-mocks (~> 3.13.0)
|
|
27
|
-
rspec-core (3.13.
|
|
29
|
+
rspec-core (3.13.6)
|
|
28
30
|
rspec-support (~> 3.13.0)
|
|
29
31
|
rspec-expectations (3.13.5)
|
|
30
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
31
33
|
rspec-support (~> 3.13.0)
|
|
32
|
-
rspec-mocks (3.13.
|
|
34
|
+
rspec-mocks (3.13.8)
|
|
33
35
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
34
36
|
rspec-support (~> 3.13.0)
|
|
35
|
-
rspec-support (3.13.
|
|
36
|
-
rubocop (1.
|
|
37
|
+
rspec-support (3.13.7)
|
|
38
|
+
rubocop (1.88.2)
|
|
37
39
|
json (~> 2.3)
|
|
38
40
|
language_server-protocol (~> 3.17.0.2)
|
|
39
41
|
lint_roller (~> 1.1.0)
|
|
40
|
-
parallel (
|
|
42
|
+
parallel (>= 1.10)
|
|
41
43
|
parser (>= 3.3.0.2)
|
|
42
44
|
rainbow (>= 2.2.2, < 4.0)
|
|
43
45
|
regexp_parser (>= 2.9.3, < 3.0)
|
|
44
|
-
rubocop-ast (>= 1.
|
|
46
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
45
47
|
ruby-progressbar (~> 1.7)
|
|
46
48
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
47
|
-
rubocop-ast (1.
|
|
49
|
+
rubocop-ast (1.50.0)
|
|
48
50
|
parser (>= 3.3.7.2)
|
|
49
|
-
prism (~> 1.
|
|
50
|
-
rubocop-performance (1.
|
|
51
|
+
prism (~> 1.7)
|
|
52
|
+
rubocop-performance (1.26.1)
|
|
51
53
|
lint_roller (~> 1.1)
|
|
52
54
|
rubocop (>= 1.75.0, < 2.0)
|
|
53
|
-
rubocop-ast (>= 1.
|
|
55
|
+
rubocop-ast (>= 1.47.1, < 2.0)
|
|
54
56
|
ruby-progressbar (1.13.0)
|
|
55
|
-
standard (1.
|
|
57
|
+
standard (1.56.0)
|
|
56
58
|
language_server-protocol (~> 3.17.0.2)
|
|
57
59
|
lint_roller (~> 1.0)
|
|
58
|
-
rubocop (~> 1.
|
|
60
|
+
rubocop (~> 1.88.0)
|
|
59
61
|
standard-custom (~> 1.0.0)
|
|
60
62
|
standard-performance (~> 1.8)
|
|
61
63
|
standard-custom (1.0.2)
|
|
62
64
|
lint_roller (~> 1.0)
|
|
63
65
|
rubocop (~> 1.50)
|
|
64
|
-
standard-performance (1.
|
|
66
|
+
standard-performance (1.9.0)
|
|
65
67
|
lint_roller (~> 1.1)
|
|
66
|
-
rubocop-performance (~> 1.
|
|
67
|
-
timecop (0.9.
|
|
68
|
+
rubocop-performance (~> 1.26.0)
|
|
69
|
+
timecop (0.9.11)
|
|
68
70
|
unicode-display_width (3.2.0)
|
|
69
71
|
unicode-emoji (~> 4.1)
|
|
70
72
|
unicode-emoji (4.2.0)
|
|
@@ -81,4 +83,4 @@ DEPENDENCIES
|
|
|
81
83
|
timecop (~> 0.9)
|
|
82
84
|
|
|
83
85
|
BUNDLED WITH
|
|
84
|
-
4.0.
|
|
86
|
+
4.0.16
|
data/README.md
CHANGED
|
@@ -75,6 +75,25 @@ end
|
|
|
75
75
|
|
|
76
76
|
Block capture records a single combined stdout/stderr snapshot. Exit status is a placeholder (`0`) in this mode.
|
|
77
77
|
|
|
78
|
+
### Differential Testing (Compare)
|
|
79
|
+
|
|
80
|
+
Use `Backspin.compare` when the question is "does my tool produce the same output as this reference tool?" Both commands run live and their output is compared directly - nothing is recorded to disk:
|
|
81
|
+
|
|
82
|
+
```ruby
|
|
83
|
+
result = Backspin.compare(
|
|
84
|
+
reference: ["bundle", "exec", "rspec", "spec/failing_spec.rb"],
|
|
85
|
+
actual: ["my-runner", "spec/failing_spec.rb"]
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
result.expected.stdout # reference output
|
|
89
|
+
result.actual.stdout # output from the command under test
|
|
90
|
+
result.verified? # true when they match
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Only stdout, stderr, and exit status are compared, so the two commands can differ in argv and environment without any normalization. `filter` and `matcher` work exactly as they do for `Backspin.run`, and a mismatch raises `Backspin::VerificationError` unless `raise_on_verification_failure` is disabled.
|
|
94
|
+
|
|
95
|
+
If the reference command produces no output at all - usually a sign it failed to start rather than that its output is genuinely empty - `compare` raises `Backspin::ReferenceCommandError` instead of comparing empty to empty.
|
|
96
|
+
|
|
78
97
|
### Recording Modes
|
|
79
98
|
|
|
80
99
|
Backspin supports different modes for controlling how commands are recorded and verified:
|
|
@@ -91,6 +110,26 @@ result = Backspin.run(["echo", "hello"], name: "echo_test", mode: :verify)
|
|
|
91
110
|
expect(result.verified?).to be true
|
|
92
111
|
```
|
|
93
112
|
|
|
113
|
+
### Environment Variable Mode Override
|
|
114
|
+
|
|
115
|
+
Set `BACKSPIN_MODE` to globally force a recording mode without changing any test code:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
# Re-record all fixtures
|
|
119
|
+
BACKSPIN_MODE=record bundle exec rspec
|
|
120
|
+
|
|
121
|
+
# Verify-only (CI, no accidental re-records)
|
|
122
|
+
BACKSPIN_MODE=verify bundle exec rspec
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Precedence (highest to lowest):
|
|
126
|
+
|
|
127
|
+
1. Explicit `mode:` kwarg (`:record` or `:verify`)
|
|
128
|
+
2. `BACKSPIN_MODE` environment variable
|
|
129
|
+
3. Auto-detection (record if no file exists, verify if it does)
|
|
130
|
+
|
|
131
|
+
Allowed values: `auto`, `record`, `verify` (case-insensitive). Invalid values raise `ArgumentError`.
|
|
132
|
+
|
|
94
133
|
### Record Metadata
|
|
95
134
|
|
|
96
135
|
Backspin writes records using `format_version: "4.1"` with top-level metadata:
|
|
@@ -191,7 +230,7 @@ Backspin.run(["echo", "id=123"], name: "record_only", filter: normalize_filter,
|
|
|
191
230
|
|
|
192
231
|
### Working with the Result Object
|
|
193
232
|
|
|
194
|
-
The API returns a `Backspin::BackspinResult` object with
|
|
233
|
+
The API returns a `Backspin::BackspinResult` object with details on run.
|
|
195
234
|
|
|
196
235
|
```ruby
|
|
197
236
|
result = Backspin.run(["sh", "-c", "echo out; echo err >&2; exit 42"], name: "my_test")
|
|
@@ -243,6 +282,34 @@ result = Backspin.run(["echo", "different"], name: "my_test")
|
|
|
243
282
|
Backspin.reset_configuration!
|
|
244
283
|
```
|
|
245
284
|
|
|
285
|
+
### Logging
|
|
286
|
+
|
|
287
|
+
Backspin includes a configurable logger for diagnostics. By default it is set to WARN level; but most messages are logged at DEBUG level.
|
|
288
|
+
So if you are looking for more detailed logs, you can set the logger to DEBUG level:
|
|
289
|
+
|
|
290
|
+
```ruby
|
|
291
|
+
Backspin.configure do |config|
|
|
292
|
+
config.logger = Logger.new($stdout)
|
|
293
|
+
config.logger.level = Logger::DEBUG
|
|
294
|
+
end
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
To replace the logger entirely:
|
|
298
|
+
|
|
299
|
+
```ruby
|
|
300
|
+
Backspin.configure do |config|
|
|
301
|
+
config.logger = Logger.new("log/backspin.log")
|
|
302
|
+
end
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
To disable Backspin logging entirely (for example in tests):
|
|
306
|
+
|
|
307
|
+
```ruby
|
|
308
|
+
Backspin.configure do |config|
|
|
309
|
+
config.logger = nil
|
|
310
|
+
end
|
|
311
|
+
```
|
|
312
|
+
|
|
246
313
|
### Credential Scrubbing
|
|
247
314
|
|
|
248
315
|
If the CLI interaction you are recording contains sensitive data in stdout/stderr, you should be careful to make sure it is not recorded to YAML.
|
|
@@ -274,6 +341,18 @@ Automatic scrubbing includes:
|
|
|
274
341
|
|
|
275
342
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests.
|
|
276
343
|
|
|
344
|
+
This repo also includes a decoupled full-stack fixture gem at `fixtures/projects/dummy_cli_gem` that uses Backspin the way downstream projects do. Run it with:
|
|
345
|
+
|
|
346
|
+
```bash
|
|
347
|
+
bundle exec rake full_stack:dummy_app
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
To re-record that fixture's committed YAML snapshots:
|
|
351
|
+
|
|
352
|
+
```bash
|
|
353
|
+
bundle exec rake full_stack:record_dummy_app
|
|
354
|
+
```
|
|
355
|
+
|
|
277
356
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
278
357
|
|
|
279
358
|
## Contributing
|
data/Rakefile
CHANGED
|
@@ -6,6 +6,33 @@ require "standard/rake"
|
|
|
6
6
|
|
|
7
7
|
RSpec::Core::RakeTask.new(:spec)
|
|
8
8
|
|
|
9
|
+
def run_in_fake_gem(command)
|
|
10
|
+
dummy_app_dir = File.expand_path("fixtures/projects/dummy_cli_gem", __dir__)
|
|
11
|
+
|
|
12
|
+
Bundler.with_unbundled_env do
|
|
13
|
+
Dir.chdir(dummy_app_dir) do
|
|
14
|
+
sh "bundle check || bundle install"
|
|
15
|
+
sh command
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
namespace :spec do
|
|
21
|
+
desc "Run the dummy fixture gem specs with Backspin (decoupled from main suite)"
|
|
22
|
+
task :fake_gem do
|
|
23
|
+
run_in_fake_gem("bundle exec rspec")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
desc "Re-record Backspin YAML fixtures for the dummy fixture gem"
|
|
27
|
+
task :fake_gem_record do
|
|
28
|
+
original_record_mode = ENV["RECORD_MODE"]
|
|
29
|
+
ENV["RECORD_MODE"] = "record"
|
|
30
|
+
run_in_fake_gem("bundle exec rspec")
|
|
31
|
+
ensure
|
|
32
|
+
ENV["RECORD_MODE"] = original_record_mode
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
9
36
|
task default: %i[spec standard]
|
|
10
37
|
|
|
11
38
|
load "release.rake" if File.exist?("release.rake")
|
data/backspin.gemspec
CHANGED
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
|
|
|
12
12
|
spec.description = "Backspin is a Ruby library for characterization testing of command-line interfaces. Inspired by VCR's cassette-based approach, it records and replays CLI interactions to make testing faster and more deterministic."
|
|
13
13
|
spec.homepage = "https://github.com/rsanheim/backspin"
|
|
14
14
|
spec.license = "MIT"
|
|
15
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 3.
|
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
|
|
16
16
|
|
|
17
17
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
18
18
|
spec.metadata["source_code_uri"] = spec.homepage
|
|
@@ -24,4 +24,6 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.bindir = "exe"
|
|
25
25
|
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
26
26
|
spec.require_paths = ["lib"]
|
|
27
|
+
|
|
28
|
+
spec.add_dependency "logger"
|
|
27
29
|
end
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# Compare API Plan — differential testing between two commands
|
|
2
|
+
|
|
3
|
+
Date: 2026-07-20 (revised 2026-07-25)
|
|
4
|
+
|
|
5
|
+
## Motivation
|
|
6
|
+
|
|
7
|
+
The most common way Backspin is used in the `plur` test suite is *differential
|
|
8
|
+
testing*: "does tool B produce the same output as reference tool A?" plur is a
|
|
9
|
+
Go reimplementation of a parallel RSpec runner, and its golden specs assert that
|
|
10
|
+
plur's output matches `rspec`'s own output byte-for-byte.
|
|
11
|
+
|
|
12
|
+
Backspin has no first-class API for this, so today it is expressed as an
|
|
13
|
+
implicit two-call dance with the same record name:
|
|
14
|
+
|
|
15
|
+
```ruby
|
|
16
|
+
# record rspec baseline
|
|
17
|
+
chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
|
|
18
|
+
# verify plur against it
|
|
19
|
+
result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
|
|
20
|
+
expect(result.verified?).to be(true)
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
That pattern works but has sharp edges, each of which has bitten a real plur
|
|
24
|
+
spec:
|
|
25
|
+
|
|
26
|
+
1. **Ordering is load-bearing and invisible.** The first `run` records only
|
|
27
|
+
because the record file happens not to exist yet; if it does exist, *both*
|
|
28
|
+
calls verify against the stored baseline — a different meaning than the code
|
|
29
|
+
reads like.
|
|
30
|
+
2. **A stale baseline requires a manual `rm`.** Change the fixture and the
|
|
31
|
+
committed `.yml` no longer matches the reference command, so the first call
|
|
32
|
+
fails on `reference-vs-stale-baseline`. Nothing signals that deleting the
|
|
33
|
+
snapshot is the fix.
|
|
34
|
+
3. **The command line must be neutralized by hand.** The two commands *differ*
|
|
35
|
+
on purpose (`rspec …` vs `plur …`), so specs stuff `"args" => ["[PLACEHOLDER]"]`
|
|
36
|
+
into the filter to stop the recorded command from mismatching.
|
|
37
|
+
4. **The filter is passed twice** and must be identical on both calls.
|
|
38
|
+
5. **The dance can span two examples.** plur's minitest golden test records in
|
|
39
|
+
one `it` and verifies in a *different* `it` — order-dependent across
|
|
40
|
+
examples, so it only works because RSpec runs in defined order.
|
|
41
|
+
|
|
42
|
+
The gap is small and specific: `Backspin.run` assumes the *record command* and
|
|
43
|
+
the *verify command* are the same. Differential testing needs them to differ.
|
|
44
|
+
|
|
45
|
+
## The idea
|
|
46
|
+
|
|
47
|
+
Add `Backspin.compare`, which runs a **reference** command and an **actual**
|
|
48
|
+
command and compares their (filtered) output. It is pure orchestration over
|
|
49
|
+
primitives Backspin already has — no new filtering, snapshot, diff, matcher, or
|
|
50
|
+
result code, and no record file.
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
Backspin.compare(
|
|
54
|
+
reference:, # String/Array command that defines correct output
|
|
55
|
+
actual:, # String/Array command under test
|
|
56
|
+
env: nil,
|
|
57
|
+
matcher: nil, # reused as-is
|
|
58
|
+
filter: nil # reused as-is
|
|
59
|
+
)
|
|
60
|
+
# => BackspinResult
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`reference` maps to `result.expected`, `actual` maps to `result.actual` — which
|
|
64
|
+
is exactly the existing `BackspinResult` contract (expected = baseline, actual =
|
|
65
|
+
what just ran). See `docs/backspin-result-api-sketch.md`.
|
|
66
|
+
|
|
67
|
+
## Scope: live-vs-live only
|
|
68
|
+
|
|
69
|
+
An earlier draft also proposed a snapshot-backed mode (`name:` + `mode:`), where
|
|
70
|
+
`compare` would record the reference to a `.yml` and later verify `actual`
|
|
71
|
+
against it. That is deliberately **not** in scope:
|
|
72
|
+
|
|
73
|
+
- In `mode: :verify` the `reference:` argument would be silently ignored. You
|
|
74
|
+
could change the reference command and nothing would happen until someone
|
|
75
|
+
deleted the record — sharp edge #2 wearing a new hat, plus a dead keyword arg.
|
|
76
|
+
- It roughly doubles the API surface (`name`, `mode`, `BACKSPIN_MODE`,
|
|
77
|
+
`filter_on`) for a caller that does not exist yet. plur always has `rspec`
|
|
78
|
+
available, in dev and in CI.
|
|
79
|
+
|
|
80
|
+
If a real caller shows up with a reference command that is expensive or absent
|
|
81
|
+
in CI, add it then. `Backspin.run` already covers "freeze one command's output."
|
|
82
|
+
|
|
83
|
+
## Reuse map — what each need already maps to
|
|
84
|
+
|
|
85
|
+
| Need | Existing primitive | Location |
|
|
86
|
+
|---|---|---|
|
|
87
|
+
| Run a command → stdout/stderr/status | `execute_command(command, env)` | `lib/backspin.rb` |
|
|
88
|
+
| Build a snapshot value object | `Snapshot.new(command_type: Open3::Capture3, …)` | `lib/backspin/snapshot.rb` |
|
|
89
|
+
| Apply `filter` to both sides | `CommandDiff#build_comparison_snapshot` | `lib/backspin/command_diff.rb` |
|
|
90
|
+
| Compare (default = stdout/stderr/status; custom matcher/hash) | `CommandDiff` + `Matcher` | `lib/backspin/command_diff.rb`, `lib/backspin/matcher.rb` |
|
|
91
|
+
| `verified?` / `diff` / `summary` | `CommandDiff` | `lib/backspin/command_diff.rb` |
|
|
92
|
+
| Result object (`expected`, `actual`, `verified?`, `diff`, `error_message`) | `BackspinResult` | `lib/backspin/backspin_result.rb` |
|
|
93
|
+
| Raise on mismatch (config-gated) | `raise_on_verification_failure!(result)` | `lib/backspin.rb` |
|
|
94
|
+
|
|
95
|
+
The heart of it: `CommandDiff.new(expected:, actual:, matcher:, filter:)`
|
|
96
|
+
already takes two snapshots, applies the filter to both, runs the matcher, and
|
|
97
|
+
produces `verified?`/`diff`. `compare` just has to hand it two snapshots.
|
|
98
|
+
|
|
99
|
+
## Semantics that fall out for free
|
|
100
|
+
|
|
101
|
+
- **args/env are not compared.** The default matcher only looks at stdout,
|
|
102
|
+
stderr, status (`Matcher#evaluate_default`), and `CommandDiff#diff` only diffs
|
|
103
|
+
those three. So two intentionally-different commands compare cleanly with no
|
|
104
|
+
`args` placeholder — deleting plur boilerplate.
|
|
105
|
+
- **command_type matches.** Both sides are `Open3::Capture3`, so
|
|
106
|
+
`CommandDiff#command_types_match?` is true.
|
|
107
|
+
- **Strict-by-default is preserved.** `raise_on_verification_failure!` already
|
|
108
|
+
raises `VerificationError` (with `result.diff`) unless
|
|
109
|
+
`raise_on_verification_failure = false`. `compare` reuses it verbatim.
|
|
110
|
+
|
|
111
|
+
## Guard: a broken reference must not pass
|
|
112
|
+
|
|
113
|
+
Live-vs-live has one failure mode the two-call dance did not: if the reference
|
|
114
|
+
command fails to *run at all* — wrong directory, `bundle` not on `PATH`, a load
|
|
115
|
+
error — it produces empty stdout and a non-zero status. If `actual` fails the
|
|
116
|
+
same way, `verified?` is true and the spec goes green while testing nothing.
|
|
117
|
+
|
|
118
|
+
`compare` raises `Backspin::ReferenceCommandError` when the reference command
|
|
119
|
+
produces no output on either stream. Exit status can't be the signal: plur's
|
|
120
|
+
golden specs deliberately run failing suites that exit 1.
|
|
121
|
+
|
|
122
|
+
## The new code
|
|
123
|
+
|
|
124
|
+
1. `Backspin.compare` public method (validation + orchestration + the guard).
|
|
125
|
+
2. A private `capture_command_snapshot(command, env)` wrapping
|
|
126
|
+
`execute_command` + `Snapshot.new`. This is **extracted from existing
|
|
127
|
+
duplication** — `perform_command_run` builds that same snapshot twice, so
|
|
128
|
+
pulling it out DRYs `run` as well.
|
|
129
|
+
3. One fix in `raise_on_verification_failure!`: only emit the `Record:` line
|
|
130
|
+
when there is a record path, so `compare` failures don't render a dangling
|
|
131
|
+
`Record: ` label.
|
|
132
|
+
|
|
133
|
+
No changes to `CommandDiff`, `Matcher`, `Snapshot`, `Record`, or
|
|
134
|
+
`BackspinResult`. `compare` lives beside `run`/`capture` in `lib/backspin.rb`.
|
|
135
|
+
|
|
136
|
+
### Notes
|
|
137
|
+
|
|
138
|
+
- There is no `filter_on:` keyword. `filter_on: :record` would mean "filter only
|
|
139
|
+
when persisting", and `compare` never persists — passing it would silently
|
|
140
|
+
disable filtering. Omitting the keyword makes that a plain `ArgumentError`.
|
|
141
|
+
- `record_path` is `nil` on the result, and `mode` is `:verify` (nothing was
|
|
142
|
+
recorded, so `recorded?` is false).
|
|
143
|
+
|
|
144
|
+
## Before / after (plur golden spec)
|
|
145
|
+
|
|
146
|
+
Today (`spec/integration/spec/aggregate_failure_golden_spec.rb` in plur):
|
|
147
|
+
|
|
148
|
+
```ruby
|
|
149
|
+
chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
|
|
150
|
+
result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
|
|
151
|
+
expect(result.verified?).to be(true)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
With `compare`:
|
|
155
|
+
|
|
156
|
+
```ruby
|
|
157
|
+
result = chdir(fixture) do
|
|
158
|
+
Backspin.compare(reference: rspec_cmd, actual: plur_cmd, filter: normalize)
|
|
159
|
+
end
|
|
160
|
+
expect(result.verified?).to be(true)
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
A `dir:` option to drop the `chdir` wrapper is a possible follow-up; not needed
|
|
164
|
+
here, since one wrapper now covers what took two.
|
|
165
|
+
|
|
166
|
+
## Success criteria
|
|
167
|
+
|
|
168
|
+
1. `Backspin.compare(reference:, actual:)` runs both and returns a
|
|
169
|
+
`BackspinResult` with `expected` = reference snapshot, `actual` = actual
|
|
170
|
+
snapshot, and boolean `verified?`.
|
|
171
|
+
2. On mismatch, `result.diff` / `result.error_message` are populated by the
|
|
172
|
+
existing `CommandDiff`, and `VerificationError` is raised unless
|
|
173
|
+
`raise_on_verification_failure = false`. The failure message has no empty
|
|
174
|
+
`Record:` line.
|
|
175
|
+
3. `filter` / `matcher` behave exactly as in `run` (same objects, same code
|
|
176
|
+
paths).
|
|
177
|
+
4. Two commands with different argv compare on output only — no `args`
|
|
178
|
+
normalization needed.
|
|
179
|
+
5. No record file is written or read.
|
|
180
|
+
6. A reference command that produces no output raises
|
|
181
|
+
`ReferenceCommandError` rather than comparing empty-to-empty.
|
|
182
|
+
7. No new filtering / snapshot / diff / matcher / result implementation — only a
|
|
183
|
+
`compare` entry point, the extracted `capture_command_snapshot` helper, and
|
|
184
|
+
the nil-`record_path` message fix.
|
|
185
|
+
|
|
186
|
+
## Downstream: what this buys plur
|
|
187
|
+
|
|
188
|
+
Six call-site pairs across four files collapse to a single call each:
|
|
189
|
+
`single_failure_golden_spec.rb` (three examples), `aggregate_failure_golden_spec.rb`,
|
|
190
|
+
`pending_output_spec.rb`, and `minitest_integration_spec.rb` (the cross-example
|
|
191
|
+
one). Five fixture YAMLs get deleted, along with every `"args" => [...]`
|
|
192
|
+
placeholder line. plur's other five Backspin specs are ordinary single-command
|
|
193
|
+
snapshot tests and are untouched.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: ../../..
|
|
3
|
+
specs:
|
|
4
|
+
backspin (0.13.0)
|
|
5
|
+
logger
|
|
6
|
+
|
|
7
|
+
PATH
|
|
8
|
+
remote: .
|
|
9
|
+
specs:
|
|
10
|
+
dummy_cli_gem (0.1.0)
|
|
11
|
+
|
|
12
|
+
GEM
|
|
13
|
+
remote: https://rubygems.org/
|
|
14
|
+
specs:
|
|
15
|
+
diff-lcs (1.6.2)
|
|
16
|
+
logger (1.7.0)
|
|
17
|
+
rake (13.3.1)
|
|
18
|
+
rspec (3.13.2)
|
|
19
|
+
rspec-core (~> 3.13.0)
|
|
20
|
+
rspec-expectations (~> 3.13.0)
|
|
21
|
+
rspec-mocks (~> 3.13.0)
|
|
22
|
+
rspec-core (3.13.6)
|
|
23
|
+
rspec-support (~> 3.13.0)
|
|
24
|
+
rspec-expectations (3.13.5)
|
|
25
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
26
|
+
rspec-support (~> 3.13.0)
|
|
27
|
+
rspec-mocks (3.13.7)
|
|
28
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
29
|
+
rspec-support (~> 3.13.0)
|
|
30
|
+
rspec-support (3.13.7)
|
|
31
|
+
|
|
32
|
+
PLATFORMS
|
|
33
|
+
aarch64-linux
|
|
34
|
+
ruby
|
|
35
|
+
|
|
36
|
+
DEPENDENCIES
|
|
37
|
+
backspin!
|
|
38
|
+
dummy_cli_gem!
|
|
39
|
+
rake (~> 13)
|
|
40
|
+
rspec (~> 3)
|
|
41
|
+
|
|
42
|
+
CHECKSUMS
|
|
43
|
+
backspin (0.13.0)
|
|
44
|
+
diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
|
|
45
|
+
dummy_cli_gem (0.1.0)
|
|
46
|
+
logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
|
|
47
|
+
rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
|
|
48
|
+
rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
|
|
49
|
+
rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
|
|
50
|
+
rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
|
|
51
|
+
rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c
|
|
52
|
+
rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
|
|
53
|
+
|
|
54
|
+
BUNDLED WITH
|
|
55
|
+
4.0.5
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
MIT License
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# DummyCliGem Fixture
|
|
2
|
+
|
|
3
|
+
This is a minimal fixture gem used by Backspin's full-stack integration checks.
|
|
4
|
+
|
|
5
|
+
- It shells out to standard unix utilities (`echo`, `ls`).
|
|
6
|
+
- Its RSpec suite verifies command output through `Backspin.run`.
|
|
7
|
+
- Snapshot YAML records are stored in-repo under `spec/fixtures/backspin`.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/dummy_cli_gem/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "dummy_cli_gem"
|
|
7
|
+
spec.version = DummyCliGem::VERSION
|
|
8
|
+
spec.authors = ["Backspin"]
|
|
9
|
+
spec.email = ["noreply@example.com"]
|
|
10
|
+
spec.summary = "Dummy CLI gem fixture for Backspin full-stack verification"
|
|
11
|
+
spec.description = "Fixture gem that shells out to unix utilities and is tested via Backspin snapshots."
|
|
12
|
+
spec.homepage = "https://example.com/dummy_cli_gem"
|
|
13
|
+
spec.license = "MIT"
|
|
14
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
|
|
15
|
+
|
|
16
|
+
spec.files = Dir.chdir(__dir__) do
|
|
17
|
+
Dir["lib/**/*.rb", "exe/*", "spec/**/*", "script/**/*", "README.md", "LICENSE.txt"]
|
|
18
|
+
end
|
|
19
|
+
spec.bindir = "exe"
|
|
20
|
+
spec.executables = ["dummy_cli_gem"]
|
|
21
|
+
spec.require_paths = ["lib"]
|
|
22
|
+
end
|