minitest-heat 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cf4115aeb375e2fef72d98e7f706dc023ae1e61fa9d1fc3e8f490797a31e2ae0
4
- data.tar.gz: ba968646ff2b12fa2891b47fb58d2d98330d117c2a038e29f1caef449bfbd857
3
+ metadata.gz: 7bc2ab074f84981718138cefc1c8afda89da1f281e4cd66f3b9d72ee48c47a00
4
+ data.tar.gz: db62f9c4371ad2d27eb4bce77af0cd421f91f0436f22c56d88ad3f7eb2dd8502
5
5
  SHA512:
6
- metadata.gz: 67a7e17e10fe0f75f5a5cad93bd97a7f4f87e136e6dfe6f338b0e8df0e80f83533d8e70b07e218fc0431e3e7c5bbdc2d14f71c8c3d32a98d846f276cb573f225
7
- data.tar.gz: 2a0cb94a433fe5b69bef58cd4fe24393adaad5bf69e76120b9296a3f8224b2e9a55da10e176bb0fa13e228e3aa57f0836d61fe0ca8116bd973681cc7e458912f
6
+ metadata.gz: eeaa5543c5f86c5cd6c9549a647614c985873fcb25a6f4184edf8e49fc800051542bd4075ce612d9b347fa5624d7db00cd3ae5c3a46656d674c0512f25fbfd4f
7
+ data.tar.gz: 31bc2da03e6fa9325614172ed2f5c2c5882f440c04631e5023244304d5de7fee6d015caf29d5a61c8bc66bd13d947335d50416e52b7678780283a025f1845c62
@@ -1,23 +1,94 @@
1
- name: Ruby
1
+ name: CI
2
2
 
3
- on: [push,pull_request]
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ tags-ignore:
7
+ - 'v*'
8
+ pull_request:
4
9
 
5
10
  env:
6
11
  CI: true
7
12
 
8
13
  jobs:
9
- build:
14
+ security:
15
+ name: Security
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - name: Set up Ruby
20
+ uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: '3.4'
23
+ bundler-cache: true
24
+ - name: Update vulnerability database
25
+ run: bundle exec bundle-audit update
26
+ - name: Run security audit
27
+ run: bundle exec bundle-audit check
28
+
29
+ test:
30
+ name: Test (Ruby ${{ matrix.ruby }})
10
31
  strategy:
32
+ fail-fast: false
11
33
  matrix:
12
- os: [ubuntu-latest, macos-latest]
13
- ruby: [2.5.9, 2.6.9, 2.7.5, 3.0.3, 3.1.0-preview1]
34
+ ruby: ['3.1', '3.2', '3.3', '3.4', '4.0']
14
35
  runs-on: ubuntu-latest
15
36
  steps:
16
- - uses: actions/checkout@v2
37
+ - uses: actions/checkout@v4
17
38
  - name: Set up Ruby
18
39
  uses: ruby/setup-ruby@v1
19
40
  with:
20
41
  ruby-version: ${{ matrix.ruby }}
42
+ bundler: '2.6.9'
43
+ bundler-cache: true
44
+ - name: Run tests
45
+ run: bundle exec rake test
46
+
47
+ changelog:
48
+ name: Changelog
49
+ runs-on: ubuntu-latest
50
+ steps:
51
+ - uses: actions/checkout@v4
52
+ - name: Validate changelog format
53
+ run: |
54
+ # Check [Unreleased] section exists
55
+ if ! grep -q '## \[Unreleased\]' CHANGELOG.md; then
56
+ echo "::error::CHANGELOG.md missing [Unreleased] section"
57
+ exit 1
58
+ fi
59
+
60
+ # Check version entries have dates (## [X.Y.Z] - YYYY-MM-DD)
61
+ bad_entries=$(grep -E '## \[[0-9]+\.[0-9]+\.[0-9]+\]' CHANGELOG.md | grep -v ' - [0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}' || true)
62
+ if [ -n "$bad_entries" ]; then
63
+ echo "::error::Version entries must have dates (## [X.Y.Z] - YYYY-MM-DD)"
64
+ echo "$bad_entries"
65
+ exit 1
66
+ fi
67
+
68
+ echo "Changelog format valid"
69
+
70
+ version:
71
+ name: Version
72
+ runs-on: ubuntu-latest
73
+ steps:
74
+ - uses: actions/checkout@v4
75
+ - name: Set up Ruby
76
+ uses: ruby/setup-ruby@v1
77
+ with:
78
+ ruby-version: '3.4'
21
79
  bundler-cache: true
22
- - name: Run Tests
23
- run: bundle exec rake
80
+ - name: Check version consistency
81
+ run: |
82
+ VERSION=$(ruby -r./lib/minitest/heat/version -e "puts Minitest::Heat::VERSION")
83
+
84
+ # If version has a CHANGELOG entry, it should have a date
85
+ if grep -q "\[${VERSION}\]" CHANGELOG.md; then
86
+ # Verify the version entry includes a date (YYYY-MM-DD)
87
+ if ! grep "\[${VERSION}\]" CHANGELOG.md | grep -q ' - [0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}'; then
88
+ echo "::error::Version ${VERSION} in CHANGELOG is missing release date"
89
+ exit 1
90
+ fi
91
+ echo "Version ${VERSION} has dated changelog entry"
92
+ else
93
+ echo "Version ${VERSION} not yet released (under Unreleased section)"
94
+ fi
@@ -0,0 +1,98 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*'
7
+
8
+ jobs:
9
+ validate:
10
+ name: Validate
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v4
14
+ with:
15
+ fetch-depth: 0
16
+ - name: Verify tag is on main branch
17
+ run: |
18
+ TAG_COMMIT=$(git rev-list -n 1 ${{ github.ref }})
19
+ if ! git merge-base --is-ancestor $TAG_COMMIT origin/main; then
20
+ echo "::error::Release tags must point to a commit on the main branch."
21
+ echo "::error::This ensures the code has passed CI before release."
22
+ exit 1
23
+ fi
24
+ echo "Tag points to commit on main branch"
25
+
26
+ publish:
27
+ name: Publish
28
+ needs: validate
29
+ runs-on: ubuntu-latest
30
+ environment: rubygems
31
+ permissions:
32
+ id-token: write
33
+ contents: read
34
+ steps:
35
+ - uses: actions/checkout@v4
36
+ - name: Set up Ruby
37
+ uses: ruby/setup-ruby@v1
38
+ with:
39
+ ruby-version: '3.4'
40
+ bundler-cache: true
41
+ - name: Build gem
42
+ run: gem build minitest-heat.gemspec
43
+ - name: Push to RubyGems
44
+ uses: rubygems/release-gem@v1
45
+
46
+ github-release:
47
+ name: GitHub Release
48
+ needs: publish
49
+ runs-on: ubuntu-latest
50
+ permissions:
51
+ contents: write
52
+ steps:
53
+ - uses: actions/checkout@v4
54
+ - name: Extract version from tag
55
+ id: version
56
+ run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
57
+ - name: Extract changelog for version
58
+ id: changelog
59
+ run: |
60
+ # Extract the section for this version from CHANGELOG.md
61
+ version="${{ steps.version.outputs.version }}"
62
+ changelog=$(awk -v ver="$version" '
63
+ /^## \[/ {
64
+ if (found) exit
65
+ if ($0 ~ "\\[" ver "\\]") found=1
66
+ next
67
+ }
68
+ found { print }
69
+ ' CHANGELOG.md)
70
+
71
+ # Handle multi-line output
72
+ {
73
+ echo "content<<EOF"
74
+ echo "$changelog"
75
+ echo "EOF"
76
+ } >> $GITHUB_OUTPUT
77
+ - name: Create GitHub Release
78
+ uses: softprops/action-gh-release@v2
79
+ with:
80
+ name: v${{ steps.version.outputs.version }}
81
+ body: |
82
+ ## Changes
83
+
84
+ ${{ steps.changelog.outputs.content }}
85
+
86
+ ## Installation
87
+
88
+ ```bash
89
+ gem install minitest-heat -v ${{ steps.version.outputs.version }}
90
+ ```
91
+
92
+ Or add to your Gemfile:
93
+
94
+ ```ruby
95
+ gem 'minitest-heat', '~> ${{ steps.version.outputs.version }}'
96
+ ```
97
+ draft: false
98
+ prerelease: false
data/.rubocop.yml CHANGED
@@ -1,8 +1,9 @@
1
1
  AllCops:
2
+ TargetRubyVersion: 3.1
2
3
  NewCops: enable
4
+ SuggestExtensions: false
3
5
  UseCache: true
4
6
  CacheRootDirectory: './'
5
- TargetRubyVersion: 2.5.9
6
7
  Exclude:
7
8
  - 'bin/**/*'
8
9
  - 'test/files/source.rb' # An example test file for reading source code
@@ -13,7 +14,11 @@ Layout/LineLength:
13
14
 
14
15
  # One case statement in a single method isn't complex.
15
16
  Metrics/CyclomaticComplexity:
16
- IgnoredMethods: ['case']
17
+ AllowedMethods: ['case']
18
+
19
+ # Development dependencies in gemspec is fine for this gem
20
+ Gemspec/DevelopmentDependencies:
21
+ Enabled: false
17
22
 
18
23
  # 10 is a good goal but a little draconian
19
24
  Metrics/MethodLength:
data/CHANGELOG.md CHANGED
@@ -1,6 +1,40 @@
1
1
  ## [Unreleased]
2
2
 
3
- Nothing at the moment.
3
+ ## [1.3.0] - 2026-01-29
4
+
5
+ ### Added
6
+ - JSON output mode via `--heat-json` flag for CI and tooling integration
7
+ - Expanded README documentation covering issue priority order and heat map scanning
8
+ - GitHub Actions release automation with trusted publishing to RubyGems
9
+ - Pre-release validation rake tasks (`release:preflight`, `release:check`, `release:audit`, `release:dry_run`)
10
+ - Comprehensive release documentation (RELEASING.md)
11
+
12
+ ### Fixed
13
+ - Muted text now readable on light terminal backgrounds (uses terminal default color instead of gray)
14
+ - Defensive error handling throughout to prevent exception messages from bubbling up
15
+ - Off-by-one error in backtrace line_count calculation
16
+ - Handle nil values safely in source.rb, line_parser.rb, backtrace.rb, issue.rb, and output classes
17
+ - Bare rescue clause in output.rb now catches specific exceptions
18
+
19
+ ### Changed
20
+ - Significantly improved test coverage for Output classes and Results
21
+ - Updated CI to test on Ruby 3.1, 3.2, 3.3, 3.4, and 4.0 on Ubuntu
22
+ - Made development dependencies (debug, awesome_print) optional for easier setup
23
+ - Removed deprecated codecov gem reference
24
+ - Removed outdated Travis CI configuration (GitHub Actions is now the primary CI)
25
+ - Updated GitHub Actions to use checkout@v4
26
+ - Updated Gemfile.lock so CI uses the latest patched dependency versions
27
+
28
+ ## [1.2.0] - 2022-10-31
29
+
30
+ Mainly some improvements to make test failures more resilient and improve the formatting when there are issues.
31
+
32
+ - Don't consider binstubs project files when colorizing the stacktrace.
33
+ - Switch debugging from Pry to debug
34
+ - Ensure overly-long exception messages are truncated to reduce excessive scrolling
35
+ - Make backtrace display smarter about how many lines to display
36
+ - Fix bug that was incorrectly deleting the bin directory
37
+ - Prepare for better handling of "stack level too deep" traces
4
38
 
5
39
  ## [1.1.0] - 2021-12-09
6
40
 
data/Gemfile CHANGED
@@ -5,5 +5,9 @@ source 'https://rubygems.org'
5
5
  # Specify your gem's dependencies in minitest-heat.gemspec
6
6
  gemspec
7
7
 
8
+ gem 'bundler-audit', '>= 0.9'
8
9
  gem 'minitest', '~> 5.0'
9
- gem 'rake', '~> 12.0'
10
+ gem 'rake', '>= 13.0'
11
+
12
+ # Constrain erb for Ruby 3.1 compatibility (erb 5.0+ requires Ruby 3.2+)
13
+ gem 'erb', '< 5.0'
data/Gemfile.lock CHANGED
@@ -1,67 +1,107 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- minitest-heat (1.1.0)
4
+ minitest-heat (1.3.0)
5
5
  minitest
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
- ast (2.4.2)
10
+ ast (2.4.3)
11
11
  awesome_print (1.9.2)
12
- coderay (1.1.3)
13
- dead_end (3.0.2)
14
- docile (1.4.0)
15
- method_source (1.0.0)
16
- minitest (5.14.4)
17
- parallel (1.21.0)
18
- parser (3.0.2.0)
12
+ bundler-audit (0.9.3)
13
+ bundler (>= 1.2.0)
14
+ thor (~> 1.0)
15
+ cgi (0.5.1)
16
+ date (3.5.1)
17
+ debug (1.11.1)
18
+ irb (~> 1.10)
19
+ reline (>= 0.3.8)
20
+ docile (1.4.1)
21
+ erb (4.0.4)
22
+ cgi (>= 0.3.3)
23
+ io-console (0.8.2)
24
+ irb (1.16.0)
25
+ pp (>= 0.6.0)
26
+ rdoc (>= 4.0.0)
27
+ reline (>= 0.4.2)
28
+ json (2.18.0)
29
+ language_server-protocol (3.17.0.5)
30
+ lint_roller (1.1.0)
31
+ minitest (5.27.0)
32
+ parallel (1.27.0)
33
+ parser (3.3.10.1)
19
34
  ast (~> 2.4.1)
20
- pry (0.14.1)
21
- coderay (~> 1.1)
22
- method_source (~> 1.0)
23
- rainbow (3.0.0)
24
- rake (12.3.3)
25
- regexp_parser (2.1.1)
26
- rexml (3.2.5)
27
- rubocop (1.22.3)
35
+ racc
36
+ pp (0.6.3)
37
+ prettyprint
38
+ prettyprint (0.2.0)
39
+ prism (1.8.0)
40
+ psych (5.3.1)
41
+ date
42
+ stringio
43
+ racc (1.8.1)
44
+ rainbow (3.1.1)
45
+ rake (13.3.1)
46
+ rdoc (7.1.0)
47
+ erb
48
+ psych (>= 4.0.0)
49
+ tsort
50
+ regexp_parser (2.11.3)
51
+ reline (0.6.3)
52
+ io-console (~> 0.5)
53
+ rubocop (1.84.0)
54
+ json (~> 2.3)
55
+ language_server-protocol (~> 3.17.0.2)
56
+ lint_roller (~> 1.1.0)
28
57
  parallel (~> 1.10)
29
- parser (>= 3.0.0.0)
58
+ parser (>= 3.3.0.2)
30
59
  rainbow (>= 2.2.2, < 4.0)
31
- regexp_parser (>= 1.8, < 3.0)
32
- rexml
33
- rubocop-ast (>= 1.12.0, < 2.0)
60
+ regexp_parser (>= 2.9.3, < 3.0)
61
+ rubocop-ast (>= 1.49.0, < 2.0)
34
62
  ruby-progressbar (~> 1.7)
35
- unicode-display_width (>= 1.4.0, < 3.0)
36
- rubocop-ast (1.13.0)
37
- parser (>= 3.0.1.1)
38
- rubocop-minitest (0.15.2)
39
- rubocop (>= 0.90, < 2.0)
40
- rubocop-rake (0.6.0)
41
- rubocop (~> 1.0)
42
- ruby-progressbar (1.11.0)
43
- simplecov (0.21.2)
63
+ unicode-display_width (>= 2.4.0, < 4.0)
64
+ rubocop-ast (1.49.0)
65
+ parser (>= 3.3.7.2)
66
+ prism (~> 1.7)
67
+ rubocop-minitest (0.38.2)
68
+ lint_roller (~> 1.1)
69
+ rubocop (>= 1.75.0, < 2.0)
70
+ rubocop-ast (>= 1.38.0, < 2.0)
71
+ rubocop-rake (0.7.1)
72
+ lint_roller (~> 1.1)
73
+ rubocop (>= 1.72.1)
74
+ ruby-progressbar (1.13.0)
75
+ simplecov (0.22.0)
44
76
  docile (~> 1.1)
45
77
  simplecov-html (~> 0.11)
46
78
  simplecov_json_formatter (~> 0.1)
47
- simplecov-html (0.12.3)
48
- simplecov_json_formatter (0.1.3)
49
- unicode-display_width (2.1.0)
79
+ simplecov-html (0.13.2)
80
+ simplecov_json_formatter (0.1.4)
81
+ stringio (3.2.0)
82
+ thor (1.5.0)
83
+ tsort (0.2.0)
84
+ unicode-display_width (3.2.0)
85
+ unicode-emoji (~> 4.1)
86
+ unicode-emoji (4.2.0)
50
87
 
51
88
  PLATFORMS
89
+ arm64-darwin-24
52
90
  ruby
53
91
 
54
92
  DEPENDENCIES
55
93
  awesome_print
56
- dead_end
94
+ bundler-audit (>= 0.9)
95
+ debug
96
+ erb (< 5.0)
57
97
  minitest (~> 5.0)
58
98
  minitest-heat!
59
- pry
60
- rake (~> 12.0)
99
+ rake (>= 13.0)
61
100
  rubocop
62
101
  rubocop-minitest
63
102
  rubocop-rake
64
103
  simplecov
104
+ simplecov_json_formatter
65
105
 
66
106
  BUNDLED WITH
67
- 2.2.30
107
+ 2.6.9
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
- # 🔥 Minitest::Heat 🔥
2
- Minitest::Heat helps you identify problems faster so you can more efficiently resolve test failures. It does this through a few different methods.
1
+ # 🔥 Minitest Heat 🔥
2
+ Minitest Heat helps you identify problems faster so you can more efficiently resolve test failures by generating a heat map that shows where failures are concentrated.
3
3
 
4
- For a more detailed explanation of Minitest Heat with screenshots, [head over the wiki for the full story](https://github.com/garrettdimon/minitest-heat/wiki).
4
+ For a more detailed explanation of Minitest Heat with screenshots, [head over to the wiki for the full story](https://github.com/garrettdimon/minitest-heat/wiki).
5
5
 
6
6
  Or for some additional insight about priorities and how it works, this [Twitter thread](https://twitter.com/garrettdimon/status/1432703746526560266) is a good read.
7
7
 
@@ -26,24 +26,106 @@ And depending on your usage, you may need to require Minitest Heat in your test
26
26
  require 'minitest/heat'
27
27
  ```
28
28
 
29
+ ## Prioritizing Your Work
30
+
31
+ Minitest Heat surfaces the most impactful problems first so you can fix what matters without scrolling through noise.
32
+
33
+ ### Issue Priority
34
+
35
+ Issues are displayed in priority order—you'll always see the most critical problems first:
36
+
37
+ | Priority | Type | Why First |
38
+ |----------|------|-----------|
39
+ | 1 | **Errors** | Exceptions in source code—your app is broken |
40
+ | 2 | **Broken** | Exceptions in test files—fix your tests before trusting them |
41
+ | 3 | **Failures** | Assertion failures—the core of what tests catch |
42
+ | 4 | **Skipped** | Only shown when no errors/failures—a reminder, not urgent |
43
+ | 5 | **Slow/Painful** | Only shown when everything passes—performance cleanup |
44
+
45
+ You never see skips or slow tests when there are real problems. This keeps you focused on what actually needs fixing.
46
+
47
+ ### The Heat Map
48
+
49
+ After listing individual issues, the heat map shows where problems are concentrated across your codebase:
50
+
51
+ - **Files sorted by severity**—most problematic files appear first
52
+ - **Line numbers with issue counts**—`42×3` means 3 issues at line 42
53
+ - **Quick hot spot identification**—if one file keeps appearing, that's where to focus
54
+
55
+ The heat map helps you spot patterns. A single file with many issues often points to a deeper problem worth investigating rather than fixing issues one by one.
56
+
29
57
  ## Configuration
30
- Minitest Heat doesn't currently offer a significant set of configuration options, but the thresholds for "Slow" and "Painfully Slow" tests can be adjusted. By default, it considers anything over 1.0s to be 'slow' and anything over 3.0s to be 'painfully slow'.
31
58
 
32
- You can add a configuration block to your `test_helper.rb` file after the `require 'minitest/heat'` line.
59
+ Minitest Heat provides configurable thresholds for identifying slow tests. By default, tests over 1.0s are considered "slow" and tests over 3.0s are "painfully slow."
33
60
 
34
- For example:
61
+ Add a configuration block to your `test_helper.rb` after `require 'minitest/heat'`:
35
62
 
36
63
  ```ruby
37
64
  Minitest::Heat.configure do |config|
38
- config.slow_threshold = 0.01
65
+ config.slow_threshold = 1.0 # seconds
66
+ config.painfully_slow_threshold = 3.0
67
+ end
68
+ ```
69
+
70
+ ### Example: Rails Application
71
+
72
+ System tests and integration tests naturally run slower. You might use higher thresholds:
73
+
74
+ ```ruby
75
+ Minitest::Heat.configure do |config|
76
+ config.slow_threshold = 3.0
77
+ config.painfully_slow_threshold = 10.0
78
+ end
79
+ ```
80
+
81
+ ### Example: Gem Development
82
+
83
+ For a gem with fast unit tests, stricter thresholds catch performance regressions early:
84
+
85
+ ```ruby
86
+ Minitest::Heat.configure do |config|
87
+ config.slow_threshold = 0.1
39
88
  config.painfully_slow_threshold = 0.5
40
89
  end
41
90
  ```
42
91
 
92
+ ## JSON Output
93
+
94
+ For CI integration, tooling, or programmatic consumption, Minitest Heat can output results as JSON:
95
+
96
+ ```bash
97
+ bundle exec rake test TESTOPTS="--heat-json"
98
+ ```
99
+
100
+ The JSON output includes:
101
+ - **statistics** - counts by issue type (errors, failures, skipped, slow, etc.)
102
+ - **timing** - total time, tests/second, assertions/second
103
+ - **heat_map** - files with issues, sorted by severity weight
104
+ - **issues** - detailed issue data with locations and messages
105
+
106
+ Example usage:
107
+ ```bash
108
+ # Capture JSON output (stderr has progress, stdout has JSON)
109
+ bundle exec rake test TESTOPTS="--heat-json" 2>/dev/null > results.json
110
+ ```
111
+
43
112
  ## Development
44
- After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
113
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
114
+
115
+ To install this gem onto your local machine, run `bundle exec rake install`. For release instructions, see [RELEASING.md](RELEASING.md).
116
+
117
+ ### Running Tests
45
118
 
46
- To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
119
+ ```bash
120
+ # Run full test suite
121
+ bundle exec rake test
122
+
123
+ # Run a single test file
124
+ bundle exec rake test TEST=test/minitest/heat/issue_test.rb
125
+
126
+ # Run a single test method
127
+ bundle exec rake test TEST=test/minitest/heat/issue_test.rb TESTOPTS="-n /test_error_issue/"
128
+ ```
47
129
 
48
130
  ### Forcing Test Failures
49
131
  In order to easily see how Minitest Heat handles different combinations of different types of failures, the following environment variables can be used to force failures.