backspin 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +18 -6
- data/.ruby-version +1 -0
- data/CHANGELOG.md +16 -1
- data/CLAUDE.md +20 -16
- data/CONTRIBUTING.md +16 -19
- data/Gemfile.lock +3 -6
- data/MATCHERS.md +28 -136
- data/README.md +61 -124
- data/backspin.gemspec +0 -3
- data/docs/backspin-result-api-sketch.md +203 -0
- data/examples/match_on_example.rb +42 -71
- data/lib/backspin/backspin_result.rb +66 -0
- data/lib/backspin/command_diff.rb +28 -23
- data/lib/backspin/configuration.rb +1 -1
- data/lib/backspin/matcher.rb +21 -27
- data/lib/backspin/record.rb +23 -36
- data/lib/backspin/recorder.rb +54 -305
- data/lib/backspin/snapshot.rb +96 -0
- data/lib/backspin/version.rb +1 -1
- data/lib/backspin.rb +127 -94
- metadata +7 -34
- data/lib/backspin/command.rb +0 -106
- data/lib/backspin/command_result.rb +0 -58
- data/lib/backspin/record_result.rb +0 -159
data/README.md
CHANGED
|
@@ -5,19 +5,19 @@
|
|
|
5
5
|
[](https://circleci.com/gh/rsanheim/backspin)
|
|
6
6
|
[](https://github.com/rsanheim/backspin/commits/main)
|
|
7
7
|
|
|
8
|
-
Backspin records and
|
|
8
|
+
Backspin records command output and block output in Ruby for easy snapshot testing of command-line interfaces. It supports direct command runs via `Open3.capture3` and block capture for more complex scenarios.
|
|
9
9
|
|
|
10
10
|
**NOTE:** Backspin should be considered alpha while pre version 1.0. It is in heavy development along-side some real-world CLI apps, so expect things to change and mature.
|
|
11
11
|
|
|
12
|
-
Inspired by [VCR](https://github.com/vcr/vcr) and other [golden master](https://en.wikipedia.org/wiki/
|
|
12
|
+
Inspired by [VCR](https://github.com/vcr/vcr) and other [characterization (aka golden master)](https://en.wikipedia.org/wiki/Characterization_test) testing libraries.
|
|
13
13
|
|
|
14
14
|
## Overview
|
|
15
15
|
|
|
16
|
-
Backspin is a Ruby library for snapshot testing (or characterization testing) of command-line interfaces. While VCR records and replays HTTP interactions, Backspin records
|
|
16
|
+
Backspin is a Ruby library for snapshot testing (or characterization testing) of command-line interfaces. While VCR records and replays HTTP interactions, Backspin records stdout, stderr, and exit status from shell commands, or captures all output from a block.
|
|
17
17
|
|
|
18
18
|
## Installation
|
|
19
19
|
|
|
20
|
-
Requires Ruby 3
|
|
20
|
+
Requires Ruby 3+.
|
|
21
21
|
|
|
22
22
|
Add this line to your application's Gemfile in the `:test` group:
|
|
23
23
|
|
|
@@ -31,7 +31,7 @@ And then run `bundle install`.
|
|
|
31
31
|
|
|
32
32
|
## Usage
|
|
33
33
|
|
|
34
|
-
### Quick Start
|
|
34
|
+
### Quick Start (Command Runs)
|
|
35
35
|
|
|
36
36
|
The simplest way to use Backspin is with the `run` method, which automatically records on the first execution and verifies on subsequent runs.
|
|
37
37
|
|
|
@@ -39,23 +39,41 @@ The simplest way to use Backspin is with the `run` method, which automatically r
|
|
|
39
39
|
require "backspin"
|
|
40
40
|
|
|
41
41
|
# First run: records the output
|
|
42
|
-
result = Backspin.run("my_command")
|
|
43
|
-
Open3.capture3("echo hello world")
|
|
44
|
-
end
|
|
42
|
+
result = Backspin.run(["echo", "hello world"], name: "my_command")
|
|
45
43
|
|
|
46
44
|
# Subsequent runs: verifies the output matches and raises on mismatch
|
|
47
|
-
Backspin.run("my_command")
|
|
48
|
-
Open3.capture3("echo hello world") # Passes - output matches
|
|
49
|
-
end
|
|
45
|
+
Backspin.run(["echo", "hello world"], name: "my_command")
|
|
50
46
|
|
|
51
47
|
# This will raise an error automatically
|
|
52
|
-
Backspin.run("my_command")
|
|
53
|
-
|
|
48
|
+
Backspin.run(["echo", "hello mars"], name: "my_command")
|
|
49
|
+
# Raises Backspin::VerificationError because output doesn't match
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
You can also pass a string command (which invokes a shell):
|
|
53
|
+
|
|
54
|
+
```ruby
|
|
55
|
+
Backspin.run("echo hello", name: "string_command")
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Block Capture
|
|
59
|
+
|
|
60
|
+
Use block capture when you need to run multiple commands or use APIs that already write to stdout/stderr:
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
# Capture all output from the block
|
|
64
|
+
result = Backspin.run(name: "block_capture") do
|
|
65
|
+
system("echo from system")
|
|
66
|
+
puts "from puts"
|
|
67
|
+
`echo from backticks`
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Alias form
|
|
71
|
+
Backspin.capture("block_capture") do
|
|
72
|
+
puts "from capture"
|
|
54
73
|
end
|
|
55
|
-
# Raises RSpec::Expectations::ExpectationNotMetError because output doesn't match
|
|
56
74
|
```
|
|
57
75
|
|
|
58
|
-
|
|
76
|
+
Block capture records a single combined stdout/stderr snapshot. Exit status is a placeholder (`0`) in this mode.
|
|
59
77
|
|
|
60
78
|
### Recording Modes
|
|
61
79
|
|
|
@@ -63,48 +81,29 @@ Backspin supports different modes for controlling how commands are recorded and
|
|
|
63
81
|
|
|
64
82
|
```ruby
|
|
65
83
|
# Auto mode (default): Record on first run, verify on subsequent runs
|
|
66
|
-
|
|
67
|
-
Open3.capture3("echo hello")
|
|
68
|
-
end
|
|
84
|
+
Backspin.run(["echo", "hello"], name: "my_command")
|
|
69
85
|
|
|
70
86
|
# Explicit record mode: Always record, overwriting existing recordings
|
|
71
|
-
|
|
72
|
-
Open3.capture3("echo hello")
|
|
73
|
-
end
|
|
74
|
-
# This will save the output to `fixtures/backspin/echo_test.yml`.
|
|
87
|
+
Backspin.run(["echo", "hello"], name: "echo_test", mode: :record)
|
|
75
88
|
|
|
76
89
|
# Explicit verify mode: Always verify against existing recording
|
|
77
|
-
result = Backspin.run("echo_test", mode: :verify)
|
|
78
|
-
Open3.capture3("echo hello")
|
|
79
|
-
end
|
|
90
|
+
result = Backspin.run(["echo", "hello"], name: "echo_test", mode: :verify)
|
|
80
91
|
expect(result.verified?).to be true
|
|
81
|
-
|
|
82
|
-
# Playback mode: Return recorded output without running the command
|
|
83
|
-
result = Backspin.run("slow_command", mode: :playback) do
|
|
84
|
-
Open3.capture3("slow_command") # Not executed - returns recorded output
|
|
85
|
-
end
|
|
86
92
|
```
|
|
87
93
|
|
|
88
|
-
###
|
|
89
|
-
|
|
90
|
-
**NOTE:** This method is deprecated and will be removed soon.
|
|
91
|
-
|
|
92
|
-
The `run!` method is maintained for backwards compatibility and works identically to `run`. Since `run` now raises on verification failure by default, both methods behave the same way:
|
|
94
|
+
### Environment Variables
|
|
93
95
|
|
|
94
96
|
```ruby
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
Backspin.run!("echo_test") do
|
|
101
|
-
Open3.capture3("echo hello")
|
|
102
|
-
end
|
|
97
|
+
Backspin.run(
|
|
98
|
+
["ruby", "-e", "print ENV.fetch('MY_ENV_VAR')"],
|
|
99
|
+
name: "with_env",
|
|
100
|
+
env: {"MY_ENV_VAR" => "value"}
|
|
101
|
+
)
|
|
103
102
|
```
|
|
104
103
|
|
|
105
|
-
|
|
104
|
+
If `env:` is not provided, it is not passed to `Open3.capture3` and is not recorded.
|
|
106
105
|
|
|
107
|
-
### Custom
|
|
106
|
+
### Custom Matchers
|
|
108
107
|
|
|
109
108
|
For cases where full matching isn't suitable, you can override via `matcher:`. **NOTE**: If you provide
|
|
110
109
|
custom matchers, that is the only matching that will be done. Default matching is skipped if user-provided
|
|
@@ -113,14 +112,11 @@ matchers are present.
|
|
|
113
112
|
You can override the full match logic with a proc:
|
|
114
113
|
|
|
115
114
|
```ruby
|
|
116
|
-
# Match stdout and status, ignore stderr
|
|
117
115
|
my_matcher = ->(recorded, actual) {
|
|
118
|
-
recorded["stdout"] == actual["stdout"] && recorded["status"]
|
|
116
|
+
recorded["stdout"] == actual["stdout"] && recorded["status"] == actual["status"]
|
|
119
117
|
}
|
|
120
118
|
|
|
121
|
-
result = Backspin.run("my_test", matcher: {
|
|
122
|
-
Open3.capture3("echo hello")
|
|
123
|
-
end
|
|
119
|
+
result = Backspin.run(["echo", "hello"], name: "my_test", matcher: {all: my_matcher})
|
|
124
120
|
```
|
|
125
121
|
|
|
126
122
|
Or you can override specific fields:
|
|
@@ -131,42 +127,29 @@ timestamp_matcher = ->(recorded, actual) {
|
|
|
131
127
|
recorded.match?(/\d{4}-\d{2}-\d{2}/) && actual.match?(/\d{4}-\d{2}-\d{2}/)
|
|
132
128
|
}
|
|
133
129
|
|
|
134
|
-
result = Backspin.run("timestamp_test", matcher: {
|
|
135
|
-
Open3.capture3("date")
|
|
136
|
-
end
|
|
137
|
-
|
|
138
|
-
# Match version numbers in stderr
|
|
139
|
-
version_matcher = ->(recorded, actual) {
|
|
140
|
-
recorded[/v(\d+)\./, 1] == actual[/v(\d+)\./, 1]
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
result = Backspin.run("version_check", matcher: { stderr: version_matcher }) do
|
|
144
|
-
Open3.capture3("node --version")
|
|
145
|
-
end
|
|
130
|
+
result = Backspin.run(["date"], name: "timestamp_test", matcher: {stdout: timestamp_matcher})
|
|
146
131
|
```
|
|
147
132
|
|
|
148
133
|
For more matcher examples and detailed documentation, see [MATCHERS.md](MATCHERS.md).
|
|
149
134
|
|
|
150
135
|
### Working with the Result Object
|
|
151
136
|
|
|
152
|
-
The API returns a `
|
|
137
|
+
The API returns a `Backspin::BackspinResult` object with helpful methods:
|
|
153
138
|
|
|
154
139
|
```ruby
|
|
155
|
-
result = Backspin.run("my_test")
|
|
156
|
-
Open3.capture3("echo out; echo err >&2; exit 42")
|
|
157
|
-
end
|
|
140
|
+
result = Backspin.run(["sh", "-c", "echo out; echo err >&2; exit 42"], name: "my_test")
|
|
158
141
|
|
|
159
142
|
# Check the mode
|
|
160
143
|
result.recorded? # true on first run
|
|
161
144
|
result.verified? # true/false on subsequent runs, nil when recording
|
|
162
|
-
result.playback? # true in playback mode
|
|
163
145
|
|
|
164
|
-
# Access output
|
|
165
|
-
result.stdout
|
|
166
|
-
result.stderr
|
|
167
|
-
result.status
|
|
168
|
-
result.
|
|
169
|
-
result.
|
|
146
|
+
# Access output snapshots
|
|
147
|
+
result.actual.stdout # "out\n"
|
|
148
|
+
result.actual.stderr # "err\n"
|
|
149
|
+
result.actual.status # 42
|
|
150
|
+
result.expected # nil in :record mode, populated in :verify mode
|
|
151
|
+
result.success? # false (non-zero exit)
|
|
152
|
+
result.output # [stdout, stderr, status] for command runs
|
|
170
153
|
|
|
171
154
|
# Debug information
|
|
172
155
|
result.record_path # Path to the YAML file
|
|
@@ -174,47 +157,12 @@ result.error_message # Human-readable error if verification failed
|
|
|
174
157
|
result.diff # Diff between expected and actual output
|
|
175
158
|
```
|
|
176
159
|
|
|
177
|
-
### Multiple Commands
|
|
178
|
-
|
|
179
|
-
Backspin automatically records and verifies all commands executed in a block:
|
|
180
|
-
|
|
181
|
-
```ruby
|
|
182
|
-
result = Backspin.run("multi_command_test") do
|
|
183
|
-
# All of these commands will be recorded
|
|
184
|
-
version, = Open3.capture3("ruby --version")
|
|
185
|
-
files, = Open3.capture3("ls -la")
|
|
186
|
-
system("echo 'Processing...'") # Note: system doesn't capture output
|
|
187
|
-
data, stderr, = Open3.capture3("curl https://api.example.com/data")
|
|
188
|
-
|
|
189
|
-
# Return whatever you need
|
|
190
|
-
{ version: version.strip, file_count: files.lines.count, data: data }
|
|
191
|
-
end
|
|
192
|
-
|
|
193
|
-
# Access individual command results
|
|
194
|
-
result.commands.size # 4
|
|
195
|
-
result.multiple_commands? # true
|
|
196
|
-
|
|
197
|
-
# For multiple commands, use these accessors
|
|
198
|
-
result.all_stdout # Array of stdout from each command
|
|
199
|
-
result.all_stderr # Array of stderr from each command
|
|
200
|
-
result.all_status # Array of exit statuses
|
|
201
|
-
|
|
202
|
-
# Or access specific commands
|
|
203
|
-
result.commands[0].stdout # Ruby version output
|
|
204
|
-
result.commands[1].stdout # ls output
|
|
205
|
-
result.commands[2].status # system call exit status (stdout is empty)
|
|
206
|
-
result.commands[3].stderr # curl errors if any
|
|
207
|
-
```
|
|
208
|
-
|
|
209
|
-
When verifying multiple commands, Backspin ensures all commands match in the exact order they were recorded. If any command differs, you'll get a detailed error showing which commands failed.
|
|
210
|
-
|
|
211
160
|
### Configuration
|
|
212
161
|
|
|
213
162
|
You can configure Backspin's behavior globally:
|
|
214
163
|
|
|
215
164
|
```ruby
|
|
216
165
|
Backspin.configure do |config|
|
|
217
|
-
# Both run and capture methods will raise on verification failure by default
|
|
218
166
|
config.raise_on_verification_failure = false # default is true
|
|
219
167
|
config.backspin_dir = "spec/fixtures/cli_records" # default is "fixtures/backspin"
|
|
220
168
|
config.scrub_credentials = false # default is true
|
|
@@ -222,42 +170,31 @@ end
|
|
|
222
170
|
```
|
|
223
171
|
|
|
224
172
|
The `raise_on_verification_failure` setting affects both `Backspin.run` and `Backspin.capture`:
|
|
225
|
-
- When `true` (default): Both methods raise
|
|
226
|
-
- `run` raises `RSpec::Expectations::ExpectationNotMetError`
|
|
227
|
-
- `capture` raises `Backspin::VerificationError` (framework-agnostic)
|
|
173
|
+
- When `true` (default): Both methods raise `Backspin::VerificationError` on verification failure
|
|
228
174
|
- When `false`: Both methods return a result with `verified?` set to false
|
|
229
175
|
|
|
230
176
|
If you need to disable the raising behavior for a specific test, you can temporarily configure it:
|
|
231
177
|
|
|
232
178
|
```ruby
|
|
233
|
-
# Temporarily disable raising for this block
|
|
234
179
|
Backspin.configure do |config|
|
|
235
180
|
config.raise_on_verification_failure = false
|
|
236
181
|
end
|
|
237
182
|
|
|
238
|
-
result = Backspin.run("my_test")
|
|
239
|
-
Open3.capture3("echo different")
|
|
240
|
-
end
|
|
183
|
+
result = Backspin.run(["echo", "different"], name: "my_test")
|
|
241
184
|
# result.verified? will be false but won't raise
|
|
242
185
|
|
|
243
|
-
# Reset configuration
|
|
244
186
|
Backspin.reset_configuration!
|
|
245
187
|
```
|
|
246
188
|
|
|
247
189
|
### Credential Scrubbing
|
|
248
190
|
|
|
249
|
-
If the CLI interaction you are recording contains sensitive data in stdout
|
|
191
|
+
If the CLI interaction you are recording contains sensitive data in stdout/stderr, you should be careful to make sure it is not recorded to YAML.
|
|
250
192
|
|
|
251
|
-
By default, Backspin automatically tries to scrub
|
|
252
|
-
Always review your record files before commiting them to source control.
|
|
253
|
-
|
|
254
|
-
Use a tool like [trufflehog](https://github.com/trufflesecurity/trufflehog) or [gitleaks](https://github.com/gitleaks/gitleaks) run via a pre-commit to catch any sensitive data before commit.
|
|
193
|
+
By default, Backspin automatically tries to scrub common credential patterns from recorded stdout, stderr, args, and env values. Always review your record files before commiting them to source control.
|
|
255
194
|
|
|
256
195
|
```ruby
|
|
257
196
|
# This will automatically scrub AWS keys, API tokens, passwords, etc.
|
|
258
|
-
Backspin.run("aws_command")
|
|
259
|
-
Open3.capture3("aws s3 ls")
|
|
260
|
-
end
|
|
197
|
+
Backspin.run(["aws", "s3", "ls"], name: "aws_command")
|
|
261
198
|
|
|
262
199
|
# Add custom patterns to scrub
|
|
263
200
|
Backspin.configure do |config|
|
data/backspin.gemspec
CHANGED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# Backspin Result API Sketch
|
|
2
|
+
|
|
3
|
+
Date: 2026-02-11
|
|
4
|
+
Branch: `spike-backspin-result-api`
|
|
5
|
+
|
|
6
|
+
## Goals
|
|
7
|
+
|
|
8
|
+
- Keep the public API small and predictable.
|
|
9
|
+
- Make runtime output and baseline output explicit.
|
|
10
|
+
- Remove multi-command semantics from the result object.
|
|
11
|
+
- Keep command run and block capture under one consistent return type.
|
|
12
|
+
|
|
13
|
+
## Public API
|
|
14
|
+
|
|
15
|
+
### Entry points
|
|
16
|
+
|
|
17
|
+
```ruby
|
|
18
|
+
Backspin.run(command = nil, name:, env: nil, mode: :auto, matcher: nil, filter: nil, &block)
|
|
19
|
+
Backspin.capture(name, mode: :auto, matcher: nil, filter: nil, &block)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Both return `BackspinResult`.
|
|
23
|
+
|
|
24
|
+
### `BackspinResult`
|
|
25
|
+
|
|
26
|
+
Top-level aggregate with one responsibility: represent this run and its comparison.
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
class BackspinResult
|
|
30
|
+
attr_reader :mode, :record_path, :actual, :expected
|
|
31
|
+
|
|
32
|
+
def recorded?; end
|
|
33
|
+
def verified?; end
|
|
34
|
+
def diff; end
|
|
35
|
+
def error_message; end
|
|
36
|
+
def success?; end
|
|
37
|
+
def failure?; end
|
|
38
|
+
end
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Rules:
|
|
42
|
+
|
|
43
|
+
- `actual` is always present and represents what just ran.
|
|
44
|
+
- `expected` is baseline snapshot when one exists.
|
|
45
|
+
- In `:record` mode, `expected` is `nil`, `verified?` is `nil`.
|
|
46
|
+
- In `:verify` mode, `expected` is present, `verified?` is boolean.
|
|
47
|
+
|
|
48
|
+
### `Snapshot`
|
|
49
|
+
|
|
50
|
+
Value object for one recorded/captured execution.
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
class Snapshot
|
|
54
|
+
attr_reader :command_type, :args, :env, :stdout, :stderr, :status, :recorded_at
|
|
55
|
+
|
|
56
|
+
def success?; end
|
|
57
|
+
def failure?; end
|
|
58
|
+
end
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Notes:
|
|
62
|
+
|
|
63
|
+
- `command_type` is `Open3::Capture3` for command runs.
|
|
64
|
+
- `command_type` is `Backspin::Capturer` for block capture.
|
|
65
|
+
- Capture status remains placeholder `0`.
|
|
66
|
+
|
|
67
|
+
## Usage Examples
|
|
68
|
+
|
|
69
|
+
### Command verify mismatch
|
|
70
|
+
|
|
71
|
+
```ruby
|
|
72
|
+
result = Backspin.run(["echo", "changed"], name: "echo_case", mode: :verify)
|
|
73
|
+
|
|
74
|
+
result.actual.stdout # "changed\n"
|
|
75
|
+
result.expected.stdout # "original\n"
|
|
76
|
+
result.verified? # false
|
|
77
|
+
result.diff # unified-ish stdout/stderr/status diff
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### First record
|
|
81
|
+
|
|
82
|
+
```ruby
|
|
83
|
+
result = Backspin.run(["echo", "hello"], name: "hello_case")
|
|
84
|
+
|
|
85
|
+
result.mode # :record
|
|
86
|
+
result.actual.stdout # "hello\n"
|
|
87
|
+
result.expected # nil
|
|
88
|
+
result.verified? # nil
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Capture verify
|
|
92
|
+
|
|
93
|
+
```ruby
|
|
94
|
+
result = Backspin.capture("capture_case", mode: :verify) do
|
|
95
|
+
puts "runtime output"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
result.actual.stdout
|
|
99
|
+
result.expected.stdout
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Unix CLI examples
|
|
103
|
+
|
|
104
|
+
```ruby
|
|
105
|
+
# 1) Record + verify a simple command
|
|
106
|
+
Backspin.run(["echo", "hello"], name: "echo_hello")
|
|
107
|
+
result = Backspin.run(["echo", "hello"], name: "echo_hello")
|
|
108
|
+
result.verified? # true
|
|
109
|
+
result.actual.stdout # "hello\n"
|
|
110
|
+
result.expected.stdout # "hello\n"
|
|
111
|
+
|
|
112
|
+
# 2) Verify mismatch with a common command
|
|
113
|
+
Backspin.run(["date", "+%Y-%m-%d"], name: "today", mode: :record)
|
|
114
|
+
result = Backspin.run(["date", "+%Y-%m-%d"], name: "today", mode: :verify)
|
|
115
|
+
result.verified? # true/false depending on day change
|
|
116
|
+
|
|
117
|
+
# 3) Capture a small shell pipeline output
|
|
118
|
+
result = Backspin.capture("grep_wc") do
|
|
119
|
+
system("printf 'alpha\\nbeta\\nalpha\\n' | grep alpha | wc -l")
|
|
120
|
+
end
|
|
121
|
+
result.actual.stdout
|
|
122
|
+
|
|
123
|
+
# 4) Verify a directory listing snapshot
|
|
124
|
+
Backspin.run(["ls", "-1"], name: "project_listing", mode: :record)
|
|
125
|
+
result = Backspin.run(["ls", "-1"], name: "project_listing", mode: :verify)
|
|
126
|
+
result.actual.stdout
|
|
127
|
+
result.expected.stdout
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Matcher and Filter Semantics
|
|
131
|
+
|
|
132
|
+
- `matcher:` applies only during verify and compares `expected` vs `actual`.
|
|
133
|
+
- `filter:` applies only when writing snapshots to disk.
|
|
134
|
+
- Default match still compares stdout/stderr/status only.
|
|
135
|
+
|
|
136
|
+
## Error Semantics
|
|
137
|
+
|
|
138
|
+
- `Backspin::VerificationError` still raised by default when verification fails.
|
|
139
|
+
- Error message is generated from `BackspinResult#error_message`.
|
|
140
|
+
- Do not duplicate `diff` content in exception formatting.
|
|
141
|
+
|
|
142
|
+
## Record Format Sketch (v4)
|
|
143
|
+
|
|
144
|
+
Single-snapshot format to match single-snapshot runtime model:
|
|
145
|
+
|
|
146
|
+
```yaml
|
|
147
|
+
---
|
|
148
|
+
format_version: "4.0"
|
|
149
|
+
recorded_at: "2026-02-11T00:00:00Z"
|
|
150
|
+
snapshot:
|
|
151
|
+
command_type: "Open3::Capture3"
|
|
152
|
+
args: ["echo", "hello"]
|
|
153
|
+
env:
|
|
154
|
+
MY_VAR: value
|
|
155
|
+
stdout: "hello\n"
|
|
156
|
+
stderr: ""
|
|
157
|
+
status: 0
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
For capture snapshots:
|
|
161
|
+
|
|
162
|
+
```yaml
|
|
163
|
+
snapshot:
|
|
164
|
+
command_type: "Backspin::Capturer"
|
|
165
|
+
args: ["<captured block>"]
|
|
166
|
+
stdout: "..."
|
|
167
|
+
stderr: "..."
|
|
168
|
+
status: 0
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
## Implemented Simplifications
|
|
172
|
+
|
|
173
|
+
- Unified all run/capture return values under `BackspinResult`.
|
|
174
|
+
- Introduced `Snapshot` as the shared value object for `actual` and `expected`.
|
|
175
|
+
- Removed multi-command result semantics from the public return API.
|
|
176
|
+
- Kept `CommandDiff`, now operating directly on snapshots.
|
|
177
|
+
- Simplified persistence to one snapshot per record file.
|
|
178
|
+
|
|
179
|
+
## Current Status
|
|
180
|
+
|
|
181
|
+
Status date: 2026-02-11
|
|
182
|
+
|
|
183
|
+
1. `Snapshot` and `BackspinResult` classes are implemented and wired into runtime paths.
|
|
184
|
+
2. `Backspin.run` and `Backspin.capture` now return `BackspinResult`.
|
|
185
|
+
3. `Record` persistence moved to v4 single-snapshot format (`snapshot` key, no `commands` array).
|
|
186
|
+
4. `Matcher` and `CommandDiff` now operate on expected/actual snapshots.
|
|
187
|
+
5. Legacy result/command layering was removed from `lib/`.
|
|
188
|
+
6. Specs have been migrated to the new result contract and v4 format.
|
|
189
|
+
7. Validation is green: `66 examples, 0 failures` and Standard lint passes.
|
|
190
|
+
8. Public docs now use `result.actual` / `result.expected` terminology.
|
|
191
|
+
|
|
192
|
+
## Success Criteria
|
|
193
|
+
|
|
194
|
+
1. `Backspin.run` and `Backspin.capture` always return `BackspinResult` with `actual` populated.
|
|
195
|
+
2. In `:record` mode, `result.expected` is `nil` and `result.verified?` is `nil`.
|
|
196
|
+
3. In `:verify` mode, `result.expected` is present, `result.verified?` is boolean, and mismatch cases populate `result.diff` plus `result.error_message`.
|
|
197
|
+
4. No multi-command result API remains in the public result contract.
|
|
198
|
+
5. Snapshot object exposes a stable single-command shape: `stdout`, `stderr`, `status`, `args`, `env`, `command_type`.
|
|
199
|
+
6. Record format uses one snapshot (v4), not a commands array.
|
|
200
|
+
7. Existing strict verification behavior remains: default raises `Backspin::VerificationError`, while `raise_on_verification_failure = false` returns a failed result without raising.
|
|
201
|
+
8. End-to-end Unix command examples are covered in specs: `echo` record/verify, `ls -1` record/verify, `date` mismatch behavior (or matcher override), and captured `grep | wc` pipeline output via `Backspin.capture`.
|
|
202
|
+
9. Matcher behavior is preserved: default matching remains stdout/stderr/status, and custom `matcher:` contract (Proc, hash fields, `:all`) continues to work for both run and capture verification.
|
|
203
|
+
10. Credential scrubbing behavior is preserved: stdout/stderr/args/env are scrubbed on persistence, capture output is scrubbed, custom patterns still apply, and verification diffs/error messages do not re-expose scrubbed secrets.
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "bundler/inline"
|
|
5
|
-
|
|
5
|
+
|
|
6
|
+
# Example usage of Backspin matchers
|
|
6
7
|
|
|
7
8
|
gemfile do
|
|
8
9
|
source "https://rubygems.org"
|
|
@@ -13,24 +14,18 @@ end
|
|
|
13
14
|
puts "Example 1: Matching timestamps with custom matcher"
|
|
14
15
|
puts "-" * 50
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
result = Backspin.run("timestamp_example") do
|
|
18
|
-
Open3.capture3("date '+%Y-%m-%d %H:%M:%S'")
|
|
19
|
-
end
|
|
17
|
+
result = Backspin.run(["date", "+%Y-%m-%d %H:%M:%S"], name: "timestamp_example")
|
|
20
18
|
puts "Recorded: #{result.stdout.chomp}"
|
|
21
19
|
|
|
22
|
-
# Sleep to ensure different timestamp
|
|
23
20
|
sleep 1
|
|
24
21
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
Open3.capture3("date '+%Y-%m-%d %H:%M:%S'")
|
|
33
|
-
end
|
|
22
|
+
result = Backspin.run(["date", "+%Y-%m-%d %H:%M:%S"], name: "timestamp_example",
|
|
23
|
+
matcher: {
|
|
24
|
+
stdout: ->(recorded, actual) {
|
|
25
|
+
recorded.match?(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/) &&
|
|
26
|
+
actual.match?(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/)
|
|
27
|
+
}
|
|
28
|
+
})
|
|
34
29
|
|
|
35
30
|
puts "Current: #{result.stdout.chomp}"
|
|
36
31
|
puts "Verified: #{result.verified?}"
|
|
@@ -40,35 +35,23 @@ puts
|
|
|
40
35
|
puts "Example 2: Matching multiple fields with different patterns"
|
|
41
36
|
puts "-" * 50
|
|
42
37
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
Open3.capture3("bash", "-c", script)
|
|
51
|
-
end
|
|
38
|
+
script = <<~SH
|
|
39
|
+
echo "PID: $$"
|
|
40
|
+
echo "Error: Timeout at $(date '+%H:%M:%S')" >&2
|
|
41
|
+
exit 1
|
|
42
|
+
SH
|
|
43
|
+
|
|
44
|
+
Backspin.run(["sh", "-c", script], name: "multi_field_example")
|
|
52
45
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
[:stdout, lambda { |recorded, actual|
|
|
57
|
-
# Both should have PID format
|
|
46
|
+
result = Backspin.run(["sh", "-c", script], name: "multi_field_example",
|
|
47
|
+
matcher: {
|
|
48
|
+
stdout: ->(recorded, actual) {
|
|
58
49
|
recorded.match?(/PID: \d+/) && actual.match?(/PID: \d+/)
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
# Both should have timeout error, ignore timestamp
|
|
50
|
+
},
|
|
51
|
+
stderr: ->(recorded, actual) {
|
|
62
52
|
recorded.match?(/Error: Timeout at/) && actual.match?(/Error: Timeout at/)
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
script = <<~BASH
|
|
66
|
-
echo "PID: $$"
|
|
67
|
-
echo "Error: Timeout at $(date '+%H:%M:%S')" >&2
|
|
68
|
-
exit 1
|
|
69
|
-
BASH
|
|
70
|
-
Open3.capture3("bash", "-c", script)
|
|
71
|
-
end
|
|
53
|
+
}
|
|
54
|
+
})
|
|
72
55
|
|
|
73
56
|
puts "Stdout: #{result.stdout.chomp}"
|
|
74
57
|
puts "Stderr: #{result.stderr.chomp}"
|
|
@@ -80,37 +63,25 @@ puts
|
|
|
80
63
|
puts "Example 3: Mixed field matching"
|
|
81
64
|
puts "-" * 50
|
|
82
65
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
echo "Status: OK"
|
|
89
|
-
BASH
|
|
90
|
-
Open3.capture3("bash", "-c", script)
|
|
91
|
-
end
|
|
66
|
+
script = <<~SH
|
|
67
|
+
echo "Version: 1.2.3"
|
|
68
|
+
echo "Build: $(date +%s)"
|
|
69
|
+
echo "Status: OK"
|
|
70
|
+
SH
|
|
92
71
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
echo "Build: $(date +%s)"
|
|
107
|
-
echo "Status: OK"
|
|
108
|
-
BASH
|
|
109
|
-
Open3.capture3("bash", "-c", script)
|
|
110
|
-
end
|
|
72
|
+
Backspin.run(["sh", "-c", script], name: "mixed_matching")
|
|
73
|
+
|
|
74
|
+
result = Backspin.run(["sh", "-c", script], name: "mixed_matching",
|
|
75
|
+
matcher: {
|
|
76
|
+
stdout: ->(recorded, actual) {
|
|
77
|
+
recorded_lines = recorded.lines
|
|
78
|
+
actual_lines = actual.lines
|
|
79
|
+
|
|
80
|
+
recorded_lines[0] == actual_lines[0] &&
|
|
81
|
+
recorded_lines[1].start_with?("Build:") && actual_lines[1].start_with?("Build:") &&
|
|
82
|
+
recorded_lines[2] == actual_lines[2]
|
|
83
|
+
}
|
|
84
|
+
})
|
|
111
85
|
|
|
112
86
|
puts "Output:\n#{result.stdout}"
|
|
113
87
|
puts "Verified: #{result.verified?}"
|
|
114
|
-
|
|
115
|
-
# Cleanup
|
|
116
|
-
FileUtils.rm_rf("fixtures/backspin")
|