backspin 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +12 -4
- data/.github/workflows/release.yml +24 -0
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.standard.yml +5 -0
- data/CHANGELOG.md +13 -1
- data/CLAUDE.md +3 -1
- data/CONTRIBUTING.md +20 -0
- data/Gemfile.lock +25 -25
- data/README.md +48 -4
- data/Rakefile +0 -2
- data/backspin.gemspec +1 -1
- data/docs/compare-api-plan.md +193 -0
- data/fixtures/projects/dummy_cli_gem/Gemfile.lock +2 -2
- data/fixtures/projects/dummy_cli_gem/dummy_cli_gem.gemspec +1 -1
- data/lib/backspin/backspin_result.rb +12 -4
- data/lib/backspin/command_diff.rb +98 -9
- data/lib/backspin/configuration.rb +5 -1
- data/lib/backspin/record.rb +1 -1
- data/lib/backspin/recorder.rb +1 -1
- data/lib/backspin/version.rb +1 -1
- data/lib/backspin.rb +69 -23
- data/mise.toml +1 -0
- metadata +4 -5
- data/.gem_release.yml +0 -13
- data/release.rake +0 -105
- data/script/run_affected_tests +0 -179
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d2fd83cd35c05021e0df8e585060247f0bee2fd1034d43861bf462bbb1d67101
|
|
4
|
+
data.tar.gz: 1ee344e3596a9442d17dbbaab2aaa541d676fb2f734d13b28ad34d49948b684b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9be21d06b0087032243e2df85bfffa060fa9a8f411ce4733a125fffbdbe40ac58d2637d1f18f9b70831730917ef9aa0baff38f6021439d37c444984f7acee886
|
|
7
|
+
data.tar.gz: 2720731b166213d551c0dd58e7478e23d20637b1d4157945fcac090298ff5059fdce1c86e3e98e20087fc5bd27de52a522b35e1a124158c914c23ede4fa53dfe
|
data/.circleci/config.yml
CHANGED
|
@@ -12,14 +12,22 @@ jobs:
|
|
|
12
12
|
ruby-version:
|
|
13
13
|
type: string
|
|
14
14
|
docker:
|
|
15
|
-
- image:
|
|
15
|
+
- image: ghcr.io/jdx/mise:2026.9.15-debian
|
|
16
|
+
environment:
|
|
17
|
+
MISE_RUBY_VERSION: << parameters.ruby-version >>
|
|
18
|
+
MISE_YES: "1"
|
|
16
19
|
steps:
|
|
17
20
|
- checkout
|
|
21
|
+
- run:
|
|
22
|
+
name: Install project tools
|
|
23
|
+
command: |
|
|
24
|
+
apt-get update && apt-get install -y --no-install-recommends build-essential libyaml-dev
|
|
25
|
+
mise install
|
|
18
26
|
- ruby/install-deps:
|
|
19
|
-
key: gems-
|
|
27
|
+
key: gems-v2-mise-ruby<< parameters.ruby-version >>
|
|
20
28
|
- run:
|
|
21
|
-
name: Run specs
|
|
22
|
-
command:
|
|
29
|
+
name: Run specs with Plur
|
|
30
|
+
command: plur spec
|
|
23
31
|
- run:
|
|
24
32
|
name: Run fake gem specs
|
|
25
33
|
command: bundle exec rake spec:fake_gem
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
inputs:
|
|
6
|
+
previous-ref:
|
|
7
|
+
description: Tag or commit before the version increase (for retrying an unpublished release)
|
|
8
|
+
required: true
|
|
9
|
+
type: string
|
|
10
|
+
pull_request:
|
|
11
|
+
types: [opened, synchronize, reopened]
|
|
12
|
+
push:
|
|
13
|
+
branches: [main]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: write
|
|
17
|
+
id-token: write
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
release:
|
|
21
|
+
uses: rsanheim/.github/.github/workflows/gem-release.yml@4ced52859dd60019b930a3f2023424e77b37ee2a
|
|
22
|
+
with:
|
|
23
|
+
gem: backspin
|
|
24
|
+
ruby-version: "4.0.0"
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4
|
|
1
|
+
4
|
data/.standard.yml
CHANGED
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
parallel: true
|
|
3
3
|
format: progress
|
|
4
4
|
|
|
5
|
+
# Match the gemspec's required_ruby_version. Without this, Standard infers the
|
|
6
|
+
# target from .ruby-version (4) and flags requires that are only redundant on
|
|
7
|
+
# Ruby >= 3.5 - removing them would break the 3.x versions we still support.
|
|
8
|
+
ruby_version: 3.2
|
|
9
|
+
|
|
5
10
|
ignore:
|
|
6
11
|
- 'tmp/**/*'
|
|
7
12
|
- 'vendor/**/*'
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## 0.
|
|
3
|
+
## 0.14.0
|
|
4
|
+
* Improved verification errors with per-field summaries for stdout, stderr, and exit status, plus three lines of context around output changes. Long diffs are limited to 50 lines; set `BACKSPIN_FULL_DIFF=1` to show the full diff.
|
|
5
|
+
* Preserved custom matcher failure reasons and made missing final newlines visible in output diffs.
|
|
6
|
+
* Fixed `backspin_dir` configuration to accept String paths as well as Pathname objects.
|
|
7
|
+
|
|
8
|
+
## 0.13.0
|
|
9
|
+
* Added `Backspin.compare(reference:, actual:)` for differential testing - runs both commands live and compares their filtered output, with no record file. Only stdout, stderr, and exit status are compared, so the two commands may differ in argv and env without normalization.
|
|
10
|
+
* Added `Backspin::ReferenceCommandError`, raised when a compare's reference command produces no output at all (usually a sign it failed to start), rather than comparing empty to empty.
|
|
11
|
+
* Raised the minimum supported Ruby to 3.2 (3.1 reached end-of-life in March 2025).
|
|
12
|
+
* Fixed snapshot timestamps to be recorded as UTC rather than local time, so re-recording in a different timezone no longer produces noisy diffs.
|
|
13
|
+
* Fixed a missing `require "time"`, which could raise `NoMethodError` on `Time#iso8601` in consumers that never load the time stdlib themselves.
|
|
14
|
+
|
|
15
|
+
## 0.12.0 - 2026-02-11
|
|
4
16
|
* Added `BACKSPIN_MODE` environment variable to globally override recording mode (`auto`, `record`, `verify`).
|
|
5
17
|
* Explicit `mode:` kwarg still takes highest precedence, followed by the env var, then auto-detection.
|
|
6
18
|
* Added configurable logger to `Backspin::Configuration` (defaults to WARN level, logfmt-lite format, and can be disabled with `config.logger = nil`).
|
data/CLAUDE.md
CHANGED
|
@@ -24,9 +24,11 @@ bin/rspec spec/[file]:[line] # Run specific test
|
|
|
24
24
|
### Building and Releasing
|
|
25
25
|
```bash
|
|
26
26
|
bin/rake install # Install gem locally for testing
|
|
27
|
-
bin/rake
|
|
27
|
+
bin/rake build # Build the gem locally
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
+
Releases use a pull request that updates `lib/backspin/version.rb` and adds a matching nonempty section to `CHANGELOG.md`. Use normal premerge CI to establish readiness. After merge to `main`, the shared GitHub Actions workflow rechecks the version and changelog, invokes `rubygems/release-gem@v1` and Bundler for tagging and gem publication, then creates the GitHub release from the changelog notes. Invalid release notes prevent publication. See `CONTRIBUTING.md` for trusted publisher setup.
|
|
31
|
+
|
|
30
32
|
### Code Quality
|
|
31
33
|
```bash
|
|
32
34
|
script/lint # Run Standard Ruby linter
|
data/CONTRIBUTING.md
CHANGED
|
@@ -11,6 +11,7 @@ Note that Backspin is in early development and the API _will_ change before stab
|
|
|
11
11
|
- [Making Changes](#making-changes)
|
|
12
12
|
- [Testing](#testing)
|
|
13
13
|
- [Submitting Changes](#submitting-changes)
|
|
14
|
+
- [Release Maintainer Setup](#release-maintainer-setup)
|
|
14
15
|
- [Code Style](#code-style)
|
|
15
16
|
- [Reporting Issues](#reporting-issues)
|
|
16
17
|
- [Feature Requests](#feature-requests)
|
|
@@ -161,6 +162,25 @@ end
|
|
|
161
162
|
- Update examples in README.md if changing public APIs
|
|
162
163
|
- Ensure CI passes (tests against Ruby 3.2, 3.3, 3.4, and 4.0)
|
|
163
164
|
|
|
165
|
+
## Release Maintainer Setup
|
|
166
|
+
|
|
167
|
+
Before the first automated release, configure a RubyGems trusted publisher for the `backspin` gem using the [reusable workflow setup](https://guides.rubygems.org/trusted-publishing/#using-reusable-workflows):
|
|
168
|
+
|
|
169
|
+
- Repository owner: `rsanheim`
|
|
170
|
+
- Repository name: `backspin`
|
|
171
|
+
- Workflow filename: `gem-release.yml`
|
|
172
|
+
- Workflow Repository Owner: `rsanheim`
|
|
173
|
+
- Workflow Repository Name: `.github`
|
|
174
|
+
- Environment: `release`
|
|
175
|
+
|
|
176
|
+
The workflow filename identifies the shared reusable workflow. The caller is `.github/workflows/release.yml`, which grants GitHub OIDC permission (`id-token: write`). No long-lived RubyGems API key is needed.
|
|
177
|
+
|
|
178
|
+
Release pull requests update `lib/backspin/version.rb` and add a nonempty `## VERSION` section to `CHANGELOG.md`; the section body becomes the GitHub release notes. Keep the version change and its notes in the same pull request. Use the normal premerge CI checks to establish release readiness. Merging to `main` starts the release workflow, which rechecks the version and changelog before publishing; invalid release notes prevent publication. Ordinary merges do not publish anything.
|
|
179
|
+
|
|
180
|
+
If publication fails before the gem is uploaded, fix the cause and manually run the Release workflow on `main`, setting `previous-ref` to the tag before the version increase. Confirm the target version is still unpublished before retrying.
|
|
181
|
+
|
|
182
|
+
The shared release workflow is pinned in `.github/workflows/release.yml`. Update that commit deliberately when adopting shared release changes. Bundler's standard `build`, `install`, and `release` tasks remain available; the automated workflow is the normal publishing path.
|
|
183
|
+
|
|
164
184
|
## Code Style
|
|
165
185
|
|
|
166
186
|
Backspin uses [Standard Ruby](https://github.com/standardrb/standard) for code formatting. Run the linter before committing:
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
backspin (0.
|
|
4
|
+
backspin (0.14.0)
|
|
5
5
|
logger
|
|
6
6
|
|
|
7
7
|
GEM
|
|
@@ -9,64 +9,64 @@ GEM
|
|
|
9
9
|
specs:
|
|
10
10
|
ast (2.4.3)
|
|
11
11
|
diff-lcs (1.6.2)
|
|
12
|
-
json (2.
|
|
13
|
-
language_server-protocol (3.17.0.
|
|
12
|
+
json (2.21.1)
|
|
13
|
+
language_server-protocol (3.17.0.6)
|
|
14
14
|
lint_roller (1.1.0)
|
|
15
15
|
logger (1.7.0)
|
|
16
|
-
parallel (1.
|
|
17
|
-
parser (3.3.
|
|
16
|
+
parallel (1.28.0)
|
|
17
|
+
parser (3.3.12.0)
|
|
18
18
|
ast (~> 2.4.1)
|
|
19
19
|
racc
|
|
20
|
-
prism (1.
|
|
20
|
+
prism (1.9.0)
|
|
21
21
|
racc (1.8.1)
|
|
22
22
|
rainbow (3.1.1)
|
|
23
|
-
rake (13.
|
|
24
|
-
regexp_parser (2.
|
|
25
|
-
rspec (3.13.
|
|
23
|
+
rake (13.4.2)
|
|
24
|
+
regexp_parser (2.12.0)
|
|
25
|
+
rspec (3.13.2)
|
|
26
26
|
rspec-core (~> 3.13.0)
|
|
27
27
|
rspec-expectations (~> 3.13.0)
|
|
28
28
|
rspec-mocks (~> 3.13.0)
|
|
29
|
-
rspec-core (3.13.
|
|
29
|
+
rspec-core (3.13.6)
|
|
30
30
|
rspec-support (~> 3.13.0)
|
|
31
31
|
rspec-expectations (3.13.5)
|
|
32
32
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
33
33
|
rspec-support (~> 3.13.0)
|
|
34
|
-
rspec-mocks (3.13.
|
|
34
|
+
rspec-mocks (3.13.8)
|
|
35
35
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
36
36
|
rspec-support (~> 3.13.0)
|
|
37
|
-
rspec-support (3.13.
|
|
38
|
-
rubocop (1.
|
|
37
|
+
rspec-support (3.13.7)
|
|
38
|
+
rubocop (1.88.2)
|
|
39
39
|
json (~> 2.3)
|
|
40
40
|
language_server-protocol (~> 3.17.0.2)
|
|
41
41
|
lint_roller (~> 1.1.0)
|
|
42
|
-
parallel (
|
|
42
|
+
parallel (>= 1.10)
|
|
43
43
|
parser (>= 3.3.0.2)
|
|
44
44
|
rainbow (>= 2.2.2, < 4.0)
|
|
45
45
|
regexp_parser (>= 2.9.3, < 3.0)
|
|
46
|
-
rubocop-ast (>= 1.
|
|
46
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
47
47
|
ruby-progressbar (~> 1.7)
|
|
48
48
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
49
|
-
rubocop-ast (1.
|
|
49
|
+
rubocop-ast (1.50.0)
|
|
50
50
|
parser (>= 3.3.7.2)
|
|
51
|
-
prism (~> 1.
|
|
52
|
-
rubocop-performance (1.
|
|
51
|
+
prism (~> 1.7)
|
|
52
|
+
rubocop-performance (1.26.1)
|
|
53
53
|
lint_roller (~> 1.1)
|
|
54
54
|
rubocop (>= 1.75.0, < 2.0)
|
|
55
|
-
rubocop-ast (>= 1.
|
|
55
|
+
rubocop-ast (>= 1.47.1, < 2.0)
|
|
56
56
|
ruby-progressbar (1.13.0)
|
|
57
|
-
standard (1.
|
|
57
|
+
standard (1.56.0)
|
|
58
58
|
language_server-protocol (~> 3.17.0.2)
|
|
59
59
|
lint_roller (~> 1.0)
|
|
60
|
-
rubocop (~> 1.
|
|
60
|
+
rubocop (~> 1.88.0)
|
|
61
61
|
standard-custom (~> 1.0.0)
|
|
62
62
|
standard-performance (~> 1.8)
|
|
63
63
|
standard-custom (1.0.2)
|
|
64
64
|
lint_roller (~> 1.0)
|
|
65
65
|
rubocop (~> 1.50)
|
|
66
|
-
standard-performance (1.
|
|
66
|
+
standard-performance (1.9.0)
|
|
67
67
|
lint_roller (~> 1.1)
|
|
68
|
-
rubocop-performance (~> 1.
|
|
69
|
-
timecop (0.9.
|
|
68
|
+
rubocop-performance (~> 1.26.0)
|
|
69
|
+
timecop (0.9.11)
|
|
70
70
|
unicode-display_width (3.2.0)
|
|
71
71
|
unicode-emoji (~> 4.1)
|
|
72
72
|
unicode-emoji (4.2.0)
|
|
@@ -83,4 +83,4 @@ DEPENDENCIES
|
|
|
83
83
|
timecop (~> 0.9)
|
|
84
84
|
|
|
85
85
|
BUNDLED WITH
|
|
86
|
-
4.0.
|
|
86
|
+
4.0.16
|
data/README.md
CHANGED
|
@@ -75,6 +75,25 @@ end
|
|
|
75
75
|
|
|
76
76
|
Block capture records a single combined stdout/stderr snapshot. Exit status is a placeholder (`0`) in this mode.
|
|
77
77
|
|
|
78
|
+
### Differential Testing (Compare)
|
|
79
|
+
|
|
80
|
+
Use `Backspin.compare` when the question is "does my tool produce the same output as this reference tool?" Both commands run live and their output is compared directly - nothing is recorded to disk:
|
|
81
|
+
|
|
82
|
+
```ruby
|
|
83
|
+
result = Backspin.compare(
|
|
84
|
+
reference: ["bundle", "exec", "rspec", "spec/failing_spec.rb"],
|
|
85
|
+
actual: ["my-runner", "spec/failing_spec.rb"]
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
result.expected.stdout # reference output
|
|
89
|
+
result.actual.stdout # output from the command under test
|
|
90
|
+
result.verified? # true when they match
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Only stdout, stderr, and exit status are compared, so the two commands can differ in argv and environment without any normalization. `filter` and `matcher` work exactly as they do for `Backspin.run`, and a mismatch raises `Backspin::VerificationError` unless `raise_on_verification_failure` is disabled.
|
|
94
|
+
|
|
95
|
+
If the reference command produces no output at all - usually a sign it failed to start rather than that its output is genuinely empty - `compare` raises `Backspin::ReferenceCommandError` instead of comparing empty to empty.
|
|
96
|
+
|
|
78
97
|
### Recording Modes
|
|
79
98
|
|
|
80
99
|
Backspin supports different modes for controlling how commands are recorded and verified:
|
|
@@ -211,7 +230,7 @@ Backspin.run(["echo", "id=123"], name: "record_only", filter: normalize_filter,
|
|
|
211
230
|
|
|
212
231
|
### Working with the Result Object
|
|
213
232
|
|
|
214
|
-
The API returns a `Backspin::BackspinResult` object with
|
|
233
|
+
The API returns a `Backspin::BackspinResult` object with details on run.
|
|
215
234
|
|
|
216
235
|
```ruby
|
|
217
236
|
result = Backspin.run(["sh", "-c", "echo out; echo err >&2; exit 42"], name: "my_test")
|
|
@@ -234,6 +253,23 @@ result.error_message # Human-readable error if verification failed
|
|
|
234
253
|
result.diff # Diff between expected and actual output
|
|
235
254
|
```
|
|
236
255
|
|
|
256
|
+
### Verification diffs
|
|
257
|
+
|
|
258
|
+
Verification errors include the failure reason and a summary of changes to stdout, stderr,
|
|
259
|
+
and exit status. Output diffs show three lines of context around each change, with `...`
|
|
260
|
+
where unchanged lines are omitted. A `\ No newline at end of output` marker identifies
|
|
261
|
+
output without a final newline.
|
|
262
|
+
|
|
263
|
+
Diffs are limited to 50 lines by default. To see every diff line, run your tests with:
|
|
264
|
+
|
|
265
|
+
```sh
|
|
266
|
+
BACKSPIN_FULL_DIFF=1 bundle exec rake spec
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The limit applies to `result.diff` and the diff portion of `result.error_message`. The
|
|
270
|
+
environment variable removes the size limit; unchanged lines outside the context windows
|
|
271
|
+
are still omitted.
|
|
272
|
+
|
|
237
273
|
### Configuration
|
|
238
274
|
|
|
239
275
|
You can configure Backspin's behavior globally:
|
|
@@ -325,16 +361,24 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
|
325
361
|
This repo also includes a decoupled full-stack fixture gem at `fixtures/projects/dummy_cli_gem` that uses Backspin the way downstream projects do. Run it with:
|
|
326
362
|
|
|
327
363
|
```bash
|
|
328
|
-
bundle exec rake
|
|
364
|
+
bundle exec rake spec:fake_gem
|
|
329
365
|
```
|
|
330
366
|
|
|
331
367
|
To re-record that fixture's committed YAML snapshots:
|
|
332
368
|
|
|
333
369
|
```bash
|
|
334
|
-
bundle exec rake
|
|
370
|
+
bundle exec rake spec:fake_gem_record
|
|
335
371
|
```
|
|
336
372
|
|
|
337
|
-
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
373
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
374
|
+
|
|
375
|
+
### Releasing
|
|
376
|
+
|
|
377
|
+
Open a pull request that updates `lib/backspin/version.rb` and adds a nonempty matching version section to `CHANGELOG.md` (for example, `## 0.14.0`). CI checks the effective gemspec version and fails if a version change has no release notes.
|
|
378
|
+
|
|
379
|
+
Use the normal premerge CI checks to establish release readiness. After the pull request merges into `main`, the shared workflow rechecks the version and changelog, then invokes `rubygems/release-gem@v1`. The official action uses OIDC trusted publishing and Bundler's standard release task to build the gem, create and push `vVERSION`, and publish to RubyGems. After the action succeeds, the shared workflow creates a GitHub release using that changelog section. Invalid release notes prevent publication. Commits that leave the gem version unchanged do not trigger a release.
|
|
380
|
+
|
|
381
|
+
The release implementation lives in [`rsanheim/.github`](https://github.com/rsanheim/.github) and is pinned to a commit in `.github/workflows/release.yml`. See [the maintainer setup](CONTRIBUTING.md#release-maintainer-setup) before enabling releases.
|
|
338
382
|
|
|
339
383
|
## Contributing
|
|
340
384
|
|
data/Rakefile
CHANGED
data/backspin.gemspec
CHANGED
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
|
|
|
12
12
|
spec.description = "Backspin is a Ruby library for characterization testing of command-line interfaces. Inspired by VCR's cassette-based approach, it records and replays CLI interactions to make testing faster and more deterministic."
|
|
13
13
|
spec.homepage = "https://github.com/rsanheim/backspin"
|
|
14
14
|
spec.license = "MIT"
|
|
15
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 3.
|
|
15
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
|
|
16
16
|
|
|
17
17
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
18
18
|
spec.metadata["source_code_uri"] = spec.homepage
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# Compare API Plan — differential testing between two commands
|
|
2
|
+
|
|
3
|
+
Date: 2026-07-20 (revised 2026-07-25)
|
|
4
|
+
|
|
5
|
+
## Motivation
|
|
6
|
+
|
|
7
|
+
The most common way Backspin is used in the `plur` test suite is *differential
|
|
8
|
+
testing*: "does tool B produce the same output as reference tool A?" plur is a
|
|
9
|
+
Go reimplementation of a parallel RSpec runner, and its golden specs assert that
|
|
10
|
+
plur's output matches `rspec`'s own output byte-for-byte.
|
|
11
|
+
|
|
12
|
+
Backspin has no first-class API for this, so today it is expressed as an
|
|
13
|
+
implicit two-call dance with the same record name:
|
|
14
|
+
|
|
15
|
+
```ruby
|
|
16
|
+
# record rspec baseline
|
|
17
|
+
chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
|
|
18
|
+
# verify plur against it
|
|
19
|
+
result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
|
|
20
|
+
expect(result.verified?).to be(true)
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
That pattern works but has sharp edges, each of which has bitten a real plur
|
|
24
|
+
spec:
|
|
25
|
+
|
|
26
|
+
1. **Ordering is load-bearing and invisible.** The first `run` records only
|
|
27
|
+
because the record file happens not to exist yet; if it does exist, *both*
|
|
28
|
+
calls verify against the stored baseline — a different meaning than the code
|
|
29
|
+
reads like.
|
|
30
|
+
2. **A stale baseline requires a manual `rm`.** Change the fixture and the
|
|
31
|
+
committed `.yml` no longer matches the reference command, so the first call
|
|
32
|
+
fails on `reference-vs-stale-baseline`. Nothing signals that deleting the
|
|
33
|
+
snapshot is the fix.
|
|
34
|
+
3. **The command line must be neutralized by hand.** The two commands *differ*
|
|
35
|
+
on purpose (`rspec …` vs `plur …`), so specs stuff `"args" => ["[PLACEHOLDER]"]`
|
|
36
|
+
into the filter to stop the recorded command from mismatching.
|
|
37
|
+
4. **The filter is passed twice** and must be identical on both calls.
|
|
38
|
+
5. **The dance can span two examples.** plur's minitest golden test records in
|
|
39
|
+
one `it` and verifies in a *different* `it` — order-dependent across
|
|
40
|
+
examples, so it only works because RSpec runs in defined order.
|
|
41
|
+
|
|
42
|
+
The gap is small and specific: `Backspin.run` assumes the *record command* and
|
|
43
|
+
the *verify command* are the same. Differential testing needs them to differ.
|
|
44
|
+
|
|
45
|
+
## The idea
|
|
46
|
+
|
|
47
|
+
Add `Backspin.compare`, which runs a **reference** command and an **actual**
|
|
48
|
+
command and compares their (filtered) output. It is pure orchestration over
|
|
49
|
+
primitives Backspin already has — no new filtering, snapshot, diff, matcher, or
|
|
50
|
+
result code, and no record file.
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
Backspin.compare(
|
|
54
|
+
reference:, # String/Array command that defines correct output
|
|
55
|
+
actual:, # String/Array command under test
|
|
56
|
+
env: nil,
|
|
57
|
+
matcher: nil, # reused as-is
|
|
58
|
+
filter: nil # reused as-is
|
|
59
|
+
)
|
|
60
|
+
# => BackspinResult
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`reference` maps to `result.expected`, `actual` maps to `result.actual` — which
|
|
64
|
+
is exactly the existing `BackspinResult` contract (expected = baseline, actual =
|
|
65
|
+
what just ran). See `docs/backspin-result-api-sketch.md`.
|
|
66
|
+
|
|
67
|
+
## Scope: live-vs-live only
|
|
68
|
+
|
|
69
|
+
An earlier draft also proposed a snapshot-backed mode (`name:` + `mode:`), where
|
|
70
|
+
`compare` would record the reference to a `.yml` and later verify `actual`
|
|
71
|
+
against it. That is deliberately **not** in scope:
|
|
72
|
+
|
|
73
|
+
- In `mode: :verify` the `reference:` argument would be silently ignored. You
|
|
74
|
+
could change the reference command and nothing would happen until someone
|
|
75
|
+
deleted the record — sharp edge #2 wearing a new hat, plus a dead keyword arg.
|
|
76
|
+
- It roughly doubles the API surface (`name`, `mode`, `BACKSPIN_MODE`,
|
|
77
|
+
`filter_on`) for a caller that does not exist yet. plur always has `rspec`
|
|
78
|
+
available, in dev and in CI.
|
|
79
|
+
|
|
80
|
+
If a real caller shows up with a reference command that is expensive or absent
|
|
81
|
+
in CI, add it then. `Backspin.run` already covers "freeze one command's output."
|
|
82
|
+
|
|
83
|
+
## Reuse map — what each need already maps to
|
|
84
|
+
|
|
85
|
+
| Need | Existing primitive | Location |
|
|
86
|
+
|---|---|---|
|
|
87
|
+
| Run a command → stdout/stderr/status | `execute_command(command, env)` | `lib/backspin.rb` |
|
|
88
|
+
| Build a snapshot value object | `Snapshot.new(command_type: Open3::Capture3, …)` | `lib/backspin/snapshot.rb` |
|
|
89
|
+
| Apply `filter` to both sides | `CommandDiff#build_comparison_snapshot` | `lib/backspin/command_diff.rb` |
|
|
90
|
+
| Compare (default = stdout/stderr/status; custom matcher/hash) | `CommandDiff` + `Matcher` | `lib/backspin/command_diff.rb`, `lib/backspin/matcher.rb` |
|
|
91
|
+
| `verified?` / `diff` / `summary` | `CommandDiff` | `lib/backspin/command_diff.rb` |
|
|
92
|
+
| Result object (`expected`, `actual`, `verified?`, `diff`, `error_message`) | `BackspinResult` | `lib/backspin/backspin_result.rb` |
|
|
93
|
+
| Raise on mismatch (config-gated) | `raise_on_verification_failure!(result)` | `lib/backspin.rb` |
|
|
94
|
+
|
|
95
|
+
The heart of it: `CommandDiff.new(expected:, actual:, matcher:, filter:)`
|
|
96
|
+
already takes two snapshots, applies the filter to both, runs the matcher, and
|
|
97
|
+
produces `verified?`/`diff`. `compare` just has to hand it two snapshots.
|
|
98
|
+
|
|
99
|
+
## Semantics that fall out for free
|
|
100
|
+
|
|
101
|
+
- **args/env are not compared.** The default matcher only looks at stdout,
|
|
102
|
+
stderr, status (`Matcher#evaluate_default`), and `CommandDiff#diff` only diffs
|
|
103
|
+
those three. So two intentionally-different commands compare cleanly with no
|
|
104
|
+
`args` placeholder — deleting plur boilerplate.
|
|
105
|
+
- **command_type matches.** Both sides are `Open3::Capture3`, so
|
|
106
|
+
`CommandDiff#command_types_match?` is true.
|
|
107
|
+
- **Strict-by-default is preserved.** `raise_on_verification_failure!` already
|
|
108
|
+
raises `VerificationError` (with `result.diff`) unless
|
|
109
|
+
`raise_on_verification_failure = false`. `compare` reuses it verbatim.
|
|
110
|
+
|
|
111
|
+
## Guard: a broken reference must not pass
|
|
112
|
+
|
|
113
|
+
Live-vs-live has one failure mode the two-call dance did not: if the reference
|
|
114
|
+
command fails to *run at all* — wrong directory, `bundle` not on `PATH`, a load
|
|
115
|
+
error — it produces empty stdout and a non-zero status. If `actual` fails the
|
|
116
|
+
same way, `verified?` is true and the spec goes green while testing nothing.
|
|
117
|
+
|
|
118
|
+
`compare` raises `Backspin::ReferenceCommandError` when the reference command
|
|
119
|
+
produces no output on either stream. Exit status can't be the signal: plur's
|
|
120
|
+
golden specs deliberately run failing suites that exit 1.
|
|
121
|
+
|
|
122
|
+
## The new code
|
|
123
|
+
|
|
124
|
+
1. `Backspin.compare` public method (validation + orchestration + the guard).
|
|
125
|
+
2. A private `capture_command_snapshot(command, env)` wrapping
|
|
126
|
+
`execute_command` + `Snapshot.new`. This is **extracted from existing
|
|
127
|
+
duplication** — `perform_command_run` builds that same snapshot twice, so
|
|
128
|
+
pulling it out DRYs `run` as well.
|
|
129
|
+
3. One fix in `raise_on_verification_failure!`: only emit the `Record:` line
|
|
130
|
+
when there is a record path, so `compare` failures don't render a dangling
|
|
131
|
+
`Record: ` label.
|
|
132
|
+
|
|
133
|
+
No changes to `CommandDiff`, `Matcher`, `Snapshot`, `Record`, or
|
|
134
|
+
`BackspinResult`. `compare` lives beside `run`/`capture` in `lib/backspin.rb`.
|
|
135
|
+
|
|
136
|
+
### Notes
|
|
137
|
+
|
|
138
|
+
- There is no `filter_on:` keyword. `filter_on: :record` would mean "filter only
|
|
139
|
+
when persisting", and `compare` never persists — passing it would silently
|
|
140
|
+
disable filtering. Omitting the keyword makes that a plain `ArgumentError`.
|
|
141
|
+
- `record_path` is `nil` on the result, and `mode` is `:verify` (nothing was
|
|
142
|
+
recorded, so `recorded?` is false).
|
|
143
|
+
|
|
144
|
+
## Before / after (plur golden spec)
|
|
145
|
+
|
|
146
|
+
Today (`spec/integration/spec/aggregate_failure_golden_spec.rb` in plur):
|
|
147
|
+
|
|
148
|
+
```ruby
|
|
149
|
+
chdir(fixture) { Backspin.run(rspec_cmd, name: "x", filter: normalize) }
|
|
150
|
+
result = chdir(fixture) { Backspin.run(plur_cmd, name: "x", filter: normalize) }
|
|
151
|
+
expect(result.verified?).to be(true)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
With `compare`:
|
|
155
|
+
|
|
156
|
+
```ruby
|
|
157
|
+
result = chdir(fixture) do
|
|
158
|
+
Backspin.compare(reference: rspec_cmd, actual: plur_cmd, filter: normalize)
|
|
159
|
+
end
|
|
160
|
+
expect(result.verified?).to be(true)
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
A `dir:` option to drop the `chdir` wrapper is a possible follow-up; not needed
|
|
164
|
+
here, since one wrapper now covers what took two.
|
|
165
|
+
|
|
166
|
+
## Success criteria
|
|
167
|
+
|
|
168
|
+
1. `Backspin.compare(reference:, actual:)` runs both and returns a
|
|
169
|
+
`BackspinResult` with `expected` = reference snapshot, `actual` = actual
|
|
170
|
+
snapshot, and boolean `verified?`.
|
|
171
|
+
2. On mismatch, `result.diff` / `result.error_message` are populated by the
|
|
172
|
+
existing `CommandDiff`, and `VerificationError` is raised unless
|
|
173
|
+
`raise_on_verification_failure = false`. The failure message has no empty
|
|
174
|
+
`Record:` line.
|
|
175
|
+
3. `filter` / `matcher` behave exactly as in `run` (same objects, same code
|
|
176
|
+
paths).
|
|
177
|
+
4. Two commands with different argv compare on output only — no `args`
|
|
178
|
+
normalization needed.
|
|
179
|
+
5. No record file is written or read.
|
|
180
|
+
6. A reference command that produces no output raises
|
|
181
|
+
`ReferenceCommandError` rather than comparing empty-to-empty.
|
|
182
|
+
7. No new filtering / snapshot / diff / matcher / result implementation — only a
|
|
183
|
+
`compare` entry point, the extracted `capture_command_snapshot` helper, and
|
|
184
|
+
the nil-`record_path` message fix.
|
|
185
|
+
|
|
186
|
+
## Downstream: what this buys plur
|
|
187
|
+
|
|
188
|
+
Six call-site pairs across four files collapse to a single call each:
|
|
189
|
+
`single_failure_golden_spec.rb` (three examples), `aggregate_failure_golden_spec.rb`,
|
|
190
|
+
`pending_output_spec.rb`, and `minitest_integration_spec.rb` (the cross-example
|
|
191
|
+
one). Five fixture YAMLs get deleted, along with every `"args" => [...]`
|
|
192
|
+
placeholder line. plur's other five Backspin specs are ordinary single-command
|
|
193
|
+
snapshot tests and are untouched.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: ../../..
|
|
3
3
|
specs:
|
|
4
|
-
backspin (0.
|
|
4
|
+
backspin (0.14.0)
|
|
5
5
|
logger
|
|
6
6
|
|
|
7
7
|
PATH
|
|
@@ -40,7 +40,7 @@ DEPENDENCIES
|
|
|
40
40
|
rspec (~> 3)
|
|
41
41
|
|
|
42
42
|
CHECKSUMS
|
|
43
|
-
backspin (0.
|
|
43
|
+
backspin (0.14.0)
|
|
44
44
|
diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
|
|
45
45
|
dummy_cli_gem (0.1.0)
|
|
46
46
|
logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
|
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.description = "Fixture gem that shells out to unix utilities and is tested via Backspin snapshots."
|
|
12
12
|
spec.homepage = "https://example.com/dummy_cli_gem"
|
|
13
13
|
spec.license = "MIT"
|
|
14
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 3.
|
|
14
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
|
|
15
15
|
|
|
16
16
|
spec.files = Dir.chdir(__dir__) do
|
|
17
17
|
Dir["lib/**/*.rb", "exe/*", "spec/**/*", "script/**/*", "README.md", "LICENSE.txt"]
|
|
@@ -36,10 +36,18 @@ module Backspin
|
|
|
36
36
|
return nil unless verified? == false
|
|
37
37
|
return "Output verification failed" unless @command_diff
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
parts = ["Output verification failed:", @command_diff.summary, "", "Summary:"]
|
|
40
|
+
@command_diff.field_summary.each do |line|
|
|
41
|
+
parts << " #{line}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
diff = @command_diff.diff
|
|
45
|
+
if diff && !diff.empty?
|
|
46
|
+
parts << ""
|
|
47
|
+
parts << diff
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
parts.join("\n")
|
|
43
51
|
end
|
|
44
52
|
|
|
45
53
|
def success?
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
module Backspin
|
|
4
4
|
# Represents the difference between expected and actual snapshots.
|
|
5
5
|
class CommandDiff
|
|
6
|
+
CONTEXT_LINES = 3
|
|
7
|
+
MAX_DIFF_LINES = 50
|
|
8
|
+
|
|
6
9
|
attr_reader :expected, :actual, :matcher
|
|
7
10
|
|
|
8
11
|
def initialize(expected:, actual:, matcher: nil, filter: nil, filter_on: :both)
|
|
@@ -26,6 +29,15 @@ module Backspin
|
|
|
26
29
|
@verified = @matcher.match?
|
|
27
30
|
end
|
|
28
31
|
|
|
32
|
+
# @return [Array<String>] Per-field change status lines
|
|
33
|
+
def field_summary
|
|
34
|
+
lines = []
|
|
35
|
+
lines << stdout_field_summary
|
|
36
|
+
lines << stderr_field_summary
|
|
37
|
+
lines << status_field_summary
|
|
38
|
+
lines
|
|
39
|
+
end
|
|
40
|
+
|
|
29
41
|
# @return [String, nil] Human-readable diff if not verified
|
|
30
42
|
def diff
|
|
31
43
|
return nil if verified?
|
|
@@ -48,7 +60,8 @@ module Backspin
|
|
|
48
60
|
parts << "Exit status: expected #{expected_compare.status}, got #{actual_compare.status}"
|
|
49
61
|
end
|
|
50
62
|
|
|
51
|
-
parts.join("\n\n")
|
|
63
|
+
result = parts.join("\n\n")
|
|
64
|
+
maybe_truncate(result)
|
|
52
65
|
end
|
|
53
66
|
|
|
54
67
|
# @return [String] Single line summary for error messages
|
|
@@ -62,6 +75,30 @@ module Backspin
|
|
|
62
75
|
|
|
63
76
|
private
|
|
64
77
|
|
|
78
|
+
def stdout_field_summary
|
|
79
|
+
if expected_compare.stdout == actual_compare.stdout
|
|
80
|
+
"stdout: unchanged"
|
|
81
|
+
else
|
|
82
|
+
"stdout: changed"
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def stderr_field_summary
|
|
87
|
+
if expected_compare.stderr == actual_compare.stderr
|
|
88
|
+
"stderr: unchanged"
|
|
89
|
+
else
|
|
90
|
+
"stderr: changed"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def status_field_summary
|
|
95
|
+
if expected_compare.status == actual_compare.status
|
|
96
|
+
"status: unchanged"
|
|
97
|
+
else
|
|
98
|
+
"status: changed (expected #{expected_compare.status}, actual #{actual_compare.status})"
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
65
102
|
def command_types_match?
|
|
66
103
|
expected.command_type == actual.command_type
|
|
67
104
|
end
|
|
@@ -83,25 +120,77 @@ module Backspin
|
|
|
83
120
|
end
|
|
84
121
|
|
|
85
122
|
def generate_line_diff(expected, actual)
|
|
86
|
-
expected_lines = (expected
|
|
87
|
-
actual_lines = (actual
|
|
123
|
+
expected_lines = split_lines(expected)
|
|
124
|
+
actual_lines = split_lines(actual)
|
|
125
|
+
max_lines = [expected_lines.length, actual_lines.length].max
|
|
126
|
+
|
|
127
|
+
changed = Array.new(max_lines, false)
|
|
128
|
+
max_lines.times do |i|
|
|
129
|
+
changed[i] = (expected_lines[i] != actual_lines[i])
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
visible = Array.new(max_lines, false)
|
|
133
|
+
max_lines.times do |i|
|
|
134
|
+
next unless changed[i]
|
|
135
|
+
range_start = [i - CONTEXT_LINES, 0].max
|
|
136
|
+
range_end = [i + CONTEXT_LINES, max_lines - 1].min
|
|
137
|
+
(range_start..range_end).each { |j| visible[j] = true }
|
|
138
|
+
end
|
|
88
139
|
|
|
89
140
|
diff_lines = []
|
|
90
|
-
|
|
141
|
+
first_visible = visible.index(true)
|
|
142
|
+
diff_lines << "..." if first_visible&.positive?
|
|
143
|
+
in_hunk = false
|
|
91
144
|
|
|
92
145
|
max_lines.times do |i|
|
|
93
|
-
|
|
94
|
-
|
|
146
|
+
unless visible[i]
|
|
147
|
+
if in_hunk
|
|
148
|
+
diff_lines << "..."
|
|
149
|
+
in_hunk = false
|
|
150
|
+
end
|
|
151
|
+
next
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
in_hunk = true
|
|
95
155
|
|
|
96
|
-
if
|
|
97
|
-
diff_lines << "-#{
|
|
98
|
-
diff_lines << "+#{
|
|
156
|
+
if changed[i]
|
|
157
|
+
diff_lines << "-#{render_line(expected_lines[i])}" if i < expected_lines.length
|
|
158
|
+
diff_lines << "+#{render_line(actual_lines[i])}" if i < actual_lines.length
|
|
159
|
+
else
|
|
160
|
+
line = expected_lines[i] || actual_lines[i]
|
|
161
|
+
diff_lines << " #{render_line(line)}"
|
|
99
162
|
end
|
|
100
163
|
end
|
|
101
164
|
|
|
102
165
|
diff_lines.join("\n")
|
|
103
166
|
end
|
|
104
167
|
|
|
168
|
+
def split_lines(value)
|
|
169
|
+
(value || "").lines
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def render_line(line)
|
|
173
|
+
rendered = line.to_s.chomp
|
|
174
|
+
return rendered if line.to_s.end_with?("\n")
|
|
175
|
+
|
|
176
|
+
"#{rendered}\n\\ No newline at end of output"
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def maybe_truncate(diff_text)
|
|
180
|
+
return diff_text if full_diff?
|
|
181
|
+
|
|
182
|
+
lines = diff_text.lines
|
|
183
|
+
return diff_text if lines.length <= MAX_DIFF_LINES
|
|
184
|
+
|
|
185
|
+
truncated = lines.first(MAX_DIFF_LINES).join
|
|
186
|
+
truncated.chomp!
|
|
187
|
+
truncated + "\n(diff truncated, set BACKSPIN_FULL_DIFF=1 for full output)"
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def full_diff?
|
|
191
|
+
ENV["BACKSPIN_FULL_DIFF"] == "1"
|
|
192
|
+
end
|
|
193
|
+
|
|
105
194
|
attr_reader :expected_compare
|
|
106
195
|
|
|
107
196
|
attr_reader :actual_compare
|
|
@@ -8,7 +8,7 @@ module Backspin
|
|
|
8
8
|
class Configuration
|
|
9
9
|
attr_accessor :scrub_credentials
|
|
10
10
|
# The directory where backspin will store its files - defaults to fixtures/backspin
|
|
11
|
-
|
|
11
|
+
attr_reader :backspin_dir
|
|
12
12
|
# Whether to raise an exception when verification fails in `run`/`capture` - defaults to true
|
|
13
13
|
attr_accessor :raise_on_verification_failure
|
|
14
14
|
# Logger for Backspin diagnostics - defaults to WARN level, logfmt-lite format
|
|
@@ -24,6 +24,10 @@ module Backspin
|
|
|
24
24
|
@logger = default_logger
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
+
def backspin_dir=(path)
|
|
28
|
+
@backspin_dir = Pathname.new(path)
|
|
29
|
+
end
|
|
30
|
+
|
|
27
31
|
def add_credential_pattern(pattern)
|
|
28
32
|
@credential_patterns << pattern
|
|
29
33
|
end
|
data/lib/backspin/record.rb
CHANGED
|
@@ -44,7 +44,7 @@ module Backspin
|
|
|
44
44
|
|
|
45
45
|
def set_snapshot(snapshot)
|
|
46
46
|
@snapshot = snapshot
|
|
47
|
-
snapshot_recorded_at = snapshot.recorded_at || Time.now.iso8601
|
|
47
|
+
snapshot_recorded_at = snapshot.recorded_at || Time.now.utc.iso8601
|
|
48
48
|
@first_recorded_at ||= snapshot_recorded_at
|
|
49
49
|
@recorded_at = snapshot_recorded_at
|
|
50
50
|
self
|
data/lib/backspin/recorder.rb
CHANGED
data/lib/backspin/version.rb
CHANGED
data/lib/backspin.rb
CHANGED
|
@@ -4,6 +4,7 @@ require "yaml"
|
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "open3"
|
|
6
6
|
require "pathname"
|
|
7
|
+
require "time"
|
|
7
8
|
require "backspin/version"
|
|
8
9
|
require "backspin/configuration"
|
|
9
10
|
require "backspin/snapshot"
|
|
@@ -18,6 +19,10 @@ module Backspin
|
|
|
18
19
|
|
|
19
20
|
class RecordNotFoundError < StandardError; end
|
|
20
21
|
|
|
22
|
+
# Raised when the reference command in a compare produced no output at all,
|
|
23
|
+
# which almost always means it failed to run rather than that it is correct.
|
|
24
|
+
class ReferenceCommandError < StandardError; end
|
|
25
|
+
|
|
21
26
|
class VerificationError < StandardError
|
|
22
27
|
attr_reader :result
|
|
23
28
|
|
|
@@ -115,6 +120,49 @@ module Backspin
|
|
|
115
120
|
perform_capture(record_name, mode: mode, matcher: matcher, filter: filter, filter_on: filter_on, &block)
|
|
116
121
|
end
|
|
117
122
|
|
|
123
|
+
# Differential testing - runs a reference command and a command under test
|
|
124
|
+
# and compares their filtered output. Nothing is recorded to disk.
|
|
125
|
+
#
|
|
126
|
+
# Only stdout, stderr, and status are compared, so the two commands may
|
|
127
|
+
# differ in argv and env without any normalization.
|
|
128
|
+
#
|
|
129
|
+
# @param reference [String, Array] Command that defines the correct output
|
|
130
|
+
# @param actual [String, Array] Command under test
|
|
131
|
+
# @param env [Hash] Environment variables passed to both commands
|
|
132
|
+
# @param matcher [Proc, Hash] Custom matcher for verification
|
|
133
|
+
# @param filter [Proc] Custom filter applied to both sides before comparing
|
|
134
|
+
# @return [BackspinResult] expected = reference, actual = command under test
|
|
135
|
+
def compare(reference:, actual:, env: nil, matcher: nil, filter: nil)
|
|
136
|
+
normalized_env = normalize_env(env)
|
|
137
|
+
|
|
138
|
+
expected_snapshot, = capture_command_snapshot(reference, normalized_env)
|
|
139
|
+
if expected_snapshot.stdout.empty? && expected_snapshot.stderr.empty?
|
|
140
|
+
raise ReferenceCommandError,
|
|
141
|
+
"Reference command produced no output (exit status #{expected_snapshot.status}): #{reference.inspect}"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
actual_snapshot, = capture_command_snapshot(actual, normalized_env)
|
|
145
|
+
|
|
146
|
+
command_diff = CommandDiff.new(
|
|
147
|
+
expected: expected_snapshot,
|
|
148
|
+
actual: actual_snapshot,
|
|
149
|
+
matcher: matcher,
|
|
150
|
+
filter: filter
|
|
151
|
+
)
|
|
152
|
+
result = BackspinResult.new(
|
|
153
|
+
mode: :verify,
|
|
154
|
+
record_path: nil,
|
|
155
|
+
actual: actual_snapshot,
|
|
156
|
+
expected: expected_snapshot,
|
|
157
|
+
verified: command_diff.verified?,
|
|
158
|
+
command_diff: command_diff
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
raise_on_verification_failure!(result)
|
|
162
|
+
|
|
163
|
+
result
|
|
164
|
+
end
|
|
165
|
+
|
|
118
166
|
private
|
|
119
167
|
|
|
120
168
|
def perform_capture(record_name, mode:, matcher:, filter:, filter_on:, &block)
|
|
@@ -147,27 +195,18 @@ module Backspin
|
|
|
147
195
|
|
|
148
196
|
record = Record.load_or_create(record_path)
|
|
149
197
|
|
|
150
|
-
normalized_env =
|
|
198
|
+
normalized_env = normalize_env(env)
|
|
151
199
|
|
|
152
200
|
result = case mode
|
|
153
201
|
when :record
|
|
154
|
-
|
|
155
|
-
actual_snapshot = Snapshot.new(
|
|
156
|
-
command_type: Open3::Capture3,
|
|
157
|
-
args: command,
|
|
158
|
-
env: normalized_env,
|
|
159
|
-
stdout: stdout,
|
|
160
|
-
stderr: stderr,
|
|
161
|
-
status: status.exitstatus,
|
|
162
|
-
recorded_at: Time.now.iso8601
|
|
163
|
-
)
|
|
202
|
+
actual_snapshot, output = capture_command_snapshot(command, normalized_env)
|
|
164
203
|
record.set_snapshot(actual_snapshot)
|
|
165
204
|
record.save(filter: filter)
|
|
166
205
|
BackspinResult.new(
|
|
167
206
|
mode: :record,
|
|
168
207
|
record_path: record.path,
|
|
169
208
|
actual: actual_snapshot,
|
|
170
|
-
output:
|
|
209
|
+
output: output
|
|
171
210
|
)
|
|
172
211
|
when :verify
|
|
173
212
|
raise RecordNotFoundError, "Record not found: #{record.path}" unless record.exists?
|
|
@@ -178,15 +217,7 @@ module Backspin
|
|
|
178
217
|
raise RecordFormatError, "Invalid record format: expected Open3::Capture3 for run"
|
|
179
218
|
end
|
|
180
219
|
|
|
181
|
-
|
|
182
|
-
actual_snapshot = Snapshot.new(
|
|
183
|
-
command_type: Open3::Capture3,
|
|
184
|
-
args: command,
|
|
185
|
-
env: normalized_env,
|
|
186
|
-
stdout: stdout,
|
|
187
|
-
stderr: stderr,
|
|
188
|
-
status: status.exitstatus
|
|
189
|
-
)
|
|
220
|
+
actual_snapshot, output = capture_command_snapshot(command, normalized_env)
|
|
190
221
|
command_diff = CommandDiff.new(
|
|
191
222
|
expected: expected_snapshot,
|
|
192
223
|
actual: actual_snapshot,
|
|
@@ -201,7 +232,7 @@ module Backspin
|
|
|
201
232
|
expected: expected_snapshot,
|
|
202
233
|
verified: command_diff.verified?,
|
|
203
234
|
command_diff: command_diff,
|
|
204
|
-
output:
|
|
235
|
+
output: output
|
|
205
236
|
)
|
|
206
237
|
else
|
|
207
238
|
raise ArgumentError, "Unknown mode: #{mode}"
|
|
@@ -212,7 +243,22 @@ module Backspin
|
|
|
212
243
|
result
|
|
213
244
|
end
|
|
214
245
|
|
|
246
|
+
def capture_command_snapshot(command, env)
|
|
247
|
+
stdout, stderr, status = execute_command(command, env)
|
|
248
|
+
snapshot = Snapshot.new(
|
|
249
|
+
command_type: Open3::Capture3,
|
|
250
|
+
args: command,
|
|
251
|
+
env: env,
|
|
252
|
+
stdout: stdout,
|
|
253
|
+
stderr: stderr,
|
|
254
|
+
status: status.exitstatus,
|
|
255
|
+
recorded_at: Time.now.utc.iso8601
|
|
256
|
+
)
|
|
257
|
+
[snapshot, [stdout, stderr, status]]
|
|
258
|
+
end
|
|
259
|
+
|
|
215
260
|
def normalize_env(env)
|
|
261
|
+
return nil if env.nil?
|
|
216
262
|
raise ArgumentError, "env must be a Hash" unless env.is_a?(Hash)
|
|
217
263
|
|
|
218
264
|
env.empty? ? nil : env
|
|
@@ -234,7 +280,7 @@ module Backspin
|
|
|
234
280
|
return unless configuration.raise_on_verification_failure && result.verified? == false
|
|
235
281
|
|
|
236
282
|
error_message = "Backspin verification failed!\n"
|
|
237
|
-
error_message += "Record: #{result.record_path}\n"
|
|
283
|
+
error_message += "Record: #{result.record_path}\n" if result.record_path
|
|
238
284
|
details = result.error_message || result.diff
|
|
239
285
|
error_message += "\n#{details}" if details
|
|
240
286
|
|
data/mise.toml
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: backspin
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.14.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rob Sanheim
|
|
@@ -33,7 +33,7 @@ extensions: []
|
|
|
33
33
|
extra_rdoc_files: []
|
|
34
34
|
files:
|
|
35
35
|
- ".circleci/config.yml"
|
|
36
|
-
- ".
|
|
36
|
+
- ".github/workflows/release.yml"
|
|
37
37
|
- ".gitignore"
|
|
38
38
|
- ".rspec"
|
|
39
39
|
- ".ruby-version"
|
|
@@ -52,6 +52,7 @@ files:
|
|
|
52
52
|
- bin/rspec
|
|
53
53
|
- bin/setup
|
|
54
54
|
- docs/backspin-result-api-sketch.md
|
|
55
|
+
- docs/compare-api-plan.md
|
|
55
56
|
- examples/match_on_example.rb
|
|
56
57
|
- fixtures/backspin/.gitkeep
|
|
57
58
|
- fixtures/projects/dummy_cli_gem/.rspec
|
|
@@ -83,9 +84,7 @@ files:
|
|
|
83
84
|
- lib/backspin/snapshot.rb
|
|
84
85
|
- lib/backspin/version.rb
|
|
85
86
|
- mise.toml
|
|
86
|
-
- release.rake
|
|
87
87
|
- script/lint
|
|
88
|
-
- script/run_affected_tests
|
|
89
88
|
homepage: https://github.com/rsanheim/backspin
|
|
90
89
|
licenses:
|
|
91
90
|
- MIT
|
|
@@ -100,7 +99,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
100
99
|
requirements:
|
|
101
100
|
- - ">="
|
|
102
101
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: 3.
|
|
102
|
+
version: 3.2.0
|
|
104
103
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
104
|
requirements:
|
|
106
105
|
- - ">="
|
data/.gem_release.yml
DELETED
data/release.rake
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "bundler/gem_tasks"
|
|
4
|
-
|
|
5
|
-
# Simplified release tasks using gem-release
|
|
6
|
-
# Install with: gem install gem-release
|
|
7
|
-
# https://github.com/svenfuchs/gem-release
|
|
8
|
-
namespace :release do
|
|
9
|
-
desc "Release a new version (bump, tag, release)"
|
|
10
|
-
task :version, [:level] do |t, args|
|
|
11
|
-
level = args[:level] || "patch"
|
|
12
|
-
|
|
13
|
-
# Pre-release checks
|
|
14
|
-
# Rake::Task["release:check"].invoke
|
|
15
|
-
|
|
16
|
-
puts "\nReleasing #{level} version..."
|
|
17
|
-
|
|
18
|
-
# Use gem-release to bump, tag, and release to rubygems and github
|
|
19
|
-
sh "gem bump --version #{level}"
|
|
20
|
-
new_version = File.read("lib/backspin/version.rb").match(/VERSION = "(\d+\.\d+\.\d+)"/)[1]
|
|
21
|
-
|
|
22
|
-
sh "bundle install"
|
|
23
|
-
sh "git commit -am 'Bump version to #{new_version}'"
|
|
24
|
-
sh "git push"
|
|
25
|
-
|
|
26
|
-
sh "gem release --tag --push"
|
|
27
|
-
Rake::Task["release:github"].invoke(new_version)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
desc "Create GitHub release for specified version or current version"
|
|
31
|
-
task :github, [:version] do |t, args|
|
|
32
|
-
version = args[:version] || Backspin::VERSION
|
|
33
|
-
|
|
34
|
-
if system("which gh > /dev/null 2>&1")
|
|
35
|
-
puts "\nCreating GitHub release for v#{version}..."
|
|
36
|
-
sh "gh release create v#{version} --title 'Release v#{version}' --generate-notes"
|
|
37
|
-
else
|
|
38
|
-
puts "\nGitHub CLI not found. Create release manually at:"
|
|
39
|
-
puts "https://github.com/rsanheim/backspin/releases/new?tag=v#{version}"
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
desc "Check if ready for release"
|
|
44
|
-
task :check do
|
|
45
|
-
require "open-uri"
|
|
46
|
-
require "json"
|
|
47
|
-
|
|
48
|
-
current_version = Backspin::VERSION
|
|
49
|
-
errors = []
|
|
50
|
-
|
|
51
|
-
# Check RubyGems for latest version
|
|
52
|
-
begin
|
|
53
|
-
gem_data = JSON.parse(URI.open("https://rubygems.org/api/v1/gems/backspin.json").read)
|
|
54
|
-
latest_version = gem_data["version"]
|
|
55
|
-
|
|
56
|
-
if Gem::Version.new(current_version) <= Gem::Version.new(latest_version)
|
|
57
|
-
errors << "Current version (#{current_version}) is not greater than latest released version (#{latest_version})"
|
|
58
|
-
else
|
|
59
|
-
puts "✓ Version #{current_version} is ready for release (latest: #{latest_version})"
|
|
60
|
-
end
|
|
61
|
-
rescue => e
|
|
62
|
-
puts "⚠ Could not check RubyGems version: #{e.message}"
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# Check git status
|
|
66
|
-
if system("git diff --quiet && git diff --cached --quiet")
|
|
67
|
-
puts "✓ No uncommitted changes"
|
|
68
|
-
else
|
|
69
|
-
errors << "You have uncommitted changes"
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# Check branch
|
|
73
|
-
current_branch = `git rev-parse --abbrev-ref HEAD`.strip
|
|
74
|
-
if current_branch != "main"
|
|
75
|
-
puts "⚠ Not on main branch (currently on #{current_branch})"
|
|
76
|
-
print "Continue anyway? (y/N): "
|
|
77
|
-
response = $stdin.gets.chomp
|
|
78
|
-
errors << "Not on main branch" unless response.downcase == "y"
|
|
79
|
-
else
|
|
80
|
-
puts "✓ On main branch"
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
unless errors.empty?
|
|
84
|
-
puts "\n❌ Cannot release:"
|
|
85
|
-
errors.each { |e| puts " - #{e}" }
|
|
86
|
-
abort
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
puts "\n✅ All checks passed!"
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# Convenience tasks
|
|
94
|
-
desc "Release patch version"
|
|
95
|
-
task "release:patch" => ["release:version"]
|
|
96
|
-
|
|
97
|
-
desc "Release minor version"
|
|
98
|
-
task "release:minor" do
|
|
99
|
-
Rake::Task["release:version"].invoke("minor")
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
desc "Release major version"
|
|
103
|
-
task "release:major" do
|
|
104
|
-
Rake::Task["release:version"].invoke("major")
|
|
105
|
-
end
|
data/script/run_affected_tests
DELETED
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
require "json"
|
|
5
|
-
require "pathname"
|
|
6
|
-
require "open3"
|
|
7
|
-
|
|
8
|
-
class AffectedTestRunner
|
|
9
|
-
EXIT_SUCCESS = 0
|
|
10
|
-
EXIT_FAILURE = 2
|
|
11
|
-
|
|
12
|
-
def initialize
|
|
13
|
-
@project_dir = ENV["CLAUDE_PROJECT_DIR"]
|
|
14
|
-
validate_environment!
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def run
|
|
18
|
-
input_data = parse_input
|
|
19
|
-
file_path = extract_file_path(input_data)
|
|
20
|
-
|
|
21
|
-
return EXIT_SUCCESS unless ruby_file?(file_path)
|
|
22
|
-
|
|
23
|
-
validated_path = validate_and_normalize_path(file_path)
|
|
24
|
-
return EXIT_SUCCESS unless validated_path
|
|
25
|
-
|
|
26
|
-
test_file = determine_test_file(validated_path)
|
|
27
|
-
return EXIT_SUCCESS unless test_file
|
|
28
|
-
|
|
29
|
-
run_tests(test_file)
|
|
30
|
-
rescue => e
|
|
31
|
-
abort_to_claude("Unexpected error: #{e.message}")
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
private
|
|
35
|
-
|
|
36
|
-
def validate_environment!
|
|
37
|
-
unless @project_dir
|
|
38
|
-
abort_to_claude("CLAUDE_PROJECT_DIR environment variable not set")
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
unless Dir.exist?(@project_dir)
|
|
42
|
-
abort_to_claude("CLAUDE_PROJECT_DIR does not exist: #{@project_dir}")
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
@project_path = Pathname.new(@project_dir).realpath
|
|
46
|
-
rescue => e
|
|
47
|
-
abort_to_claude("Invalid CLAUDE_PROJECT_DIR: #{e.message}")
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def parse_input
|
|
51
|
-
input = $stdin.read
|
|
52
|
-
JSON.parse(input)
|
|
53
|
-
rescue JSON::ParserError => e
|
|
54
|
-
abort_to_claude("Invalid JSON input: #{e.message}")
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def extract_file_path(data)
|
|
58
|
-
file_path = data.dig("tool_input", "file_path")
|
|
59
|
-
|
|
60
|
-
unless file_path && !file_path.empty?
|
|
61
|
-
abort_to_claude("No file_path provided in input")
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
file_path
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
def ruby_file?(file_path)
|
|
68
|
-
file_path.end_with?(".rb")
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def validate_and_normalize_path(file_path)
|
|
72
|
-
# Expand the path to get absolute path
|
|
73
|
-
expanded_path = File.expand_path(file_path, @project_dir)
|
|
74
|
-
normalized_path = Pathname.new(expanded_path).cleanpath
|
|
75
|
-
|
|
76
|
-
# Check if the path is within the project directory
|
|
77
|
-
unless normalized_path.to_s.start_with?(@project_path.to_s)
|
|
78
|
-
log_info("File path outside project directory: #{file_path}")
|
|
79
|
-
return nil
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
# Convert back to relative path from project root for consistency
|
|
83
|
-
normalized_path.relative_path_from(@project_path).to_s
|
|
84
|
-
rescue => e
|
|
85
|
-
log_info("Invalid file path: #{file_path} - #{e.message}")
|
|
86
|
-
nil
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
def determine_test_file(file_path)
|
|
90
|
-
log_info("Determining tests for: #{file_path}")
|
|
91
|
-
|
|
92
|
-
if spec_file?(file_path)
|
|
93
|
-
# If a spec file was modified, run just that spec
|
|
94
|
-
test_file = file_path
|
|
95
|
-
log_info("Running single spec: #{test_file}")
|
|
96
|
-
elsif lib_file?(file_path)
|
|
97
|
-
# If a lib file was modified, try to find corresponding spec
|
|
98
|
-
test_file = lib_to_spec_path(file_path)
|
|
99
|
-
|
|
100
|
-
if File.exist?(File.join(@project_dir, test_file))
|
|
101
|
-
log_info("Running corresponding spec: #{test_file}")
|
|
102
|
-
else
|
|
103
|
-
# No corresponding spec found, run all tests
|
|
104
|
-
log_info("No corresponding spec found, running all tests")
|
|
105
|
-
test_file = "spec"
|
|
106
|
-
end
|
|
107
|
-
else
|
|
108
|
-
# For other Ruby files, run all tests
|
|
109
|
-
log_info("Running all tests")
|
|
110
|
-
test_file = "spec"
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
# Validate test file/directory exists
|
|
114
|
-
full_test_path = File.join(@project_dir, test_file)
|
|
115
|
-
unless File.exist?(full_test_path)
|
|
116
|
-
abort_to_claude("Test file/directory does not exist: #{test_file}")
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
test_file
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
def spec_file?(file_path)
|
|
123
|
-
file_path.match?(%r{^.*spec/.*_spec\.rb$})
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
def lib_file?(file_path)
|
|
127
|
-
file_path.match?(%r{^.*lib/.*})
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
def lib_to_spec_path(lib_path)
|
|
131
|
-
# Convert lib/backspin/foo.rb to spec/backspin/foo_spec.rb
|
|
132
|
-
lib_path.sub(%r{^(.*/)?lib/}, '\1spec/')
|
|
133
|
-
.sub(/\.rb$/, "_spec.rb")
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
def run_tests(test_file)
|
|
137
|
-
rspec_path = File.join(@project_dir, "bin", "rspec")
|
|
138
|
-
|
|
139
|
-
unless File.executable?(rspec_path)
|
|
140
|
-
abort_to_claude("rspec binary not found or not executable: #{rspec_path}")
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
log_info("Running: bin/rspec #{test_file}")
|
|
144
|
-
|
|
145
|
-
# Change to project directory for command execution
|
|
146
|
-
Dir.chdir(@project_dir) do
|
|
147
|
-
stdout, stderr, status = Open3.capture3("bin/rspec", test_file)
|
|
148
|
-
|
|
149
|
-
# Output both stdout and stderr
|
|
150
|
-
$stdout.print stdout
|
|
151
|
-
$stderr.print stderr
|
|
152
|
-
|
|
153
|
-
if status.success?
|
|
154
|
-
log_info("✅ Tests passed")
|
|
155
|
-
EXIT_SUCCESS
|
|
156
|
-
else
|
|
157
|
-
warn("❌ Tests failed - fix before continuing")
|
|
158
|
-
EXIT_FAILURE
|
|
159
|
-
end
|
|
160
|
-
end
|
|
161
|
-
rescue => e
|
|
162
|
-
abort_to_claude("Failed to run tests: #{e.message}")
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
def log_info(message)
|
|
166
|
-
warn(message)
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
def abort_to_claude(message)
|
|
170
|
-
warn("❌ ERROR: #{message}")
|
|
171
|
-
exit EXIT_FAILURE
|
|
172
|
-
end
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
# Run the script
|
|
176
|
-
if __FILE__ == $0
|
|
177
|
-
runner = AffectedTestRunner.new
|
|
178
|
-
exit runner.run
|
|
179
|
-
end
|