minitest-smartdiff 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.travis.yml +7 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +56 -0
- data/README.md +90 -0
- data/Rakefile +10 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/minitest/smartdiff/version.rb +5 -0
- data/lib/minitest/smartdiff.rb +177 -0
- data/minitest-smartdiff.gemspec +33 -0
- metadata +139 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: abc313dca9e7330ddc1104fb2ae691b5cb7725fe02b36291b1a83ad9e69b2b33
|
4
|
+
data.tar.gz: 5d5bc8a66463630af71e9f7ecfd33a0c69f84adb479ddc5df04abf06873f2d80
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a401a5d70f5ad5388aaf9c02cacb1b909341b28e7ab6e66cefff0756c8e53dba9b35f6ce702cc75b435e43a874cd537a7b700b2b73ae2c697a9b70fe5074a03b
|
7
|
+
data.tar.gz: c0d7f89296dcd2d32cbcd0cf248764e776cd519fca5cd5b29b5b84eaa3bd37de676eb9a800d47419eb6dac41a39b949879373394cd77e1aff09547b694bf27c5
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
minitest-smartdiff (0.1.0)
|
5
|
+
ruby-openai (~> 7.0)
|
6
|
+
xxhash (~> 0.5)
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: https://rubygems.org/
|
10
|
+
specs:
|
11
|
+
debug (1.9.2)
|
12
|
+
irb (~> 1.10)
|
13
|
+
reline (>= 0.3.8)
|
14
|
+
event_stream_parser (1.0.0)
|
15
|
+
faraday (2.9.0)
|
16
|
+
faraday-net_http (>= 2.0, < 3.2)
|
17
|
+
faraday-multipart (1.0.4)
|
18
|
+
multipart-post (~> 2)
|
19
|
+
faraday-net_http (3.1.0)
|
20
|
+
net-http
|
21
|
+
io-console (0.7.2)
|
22
|
+
irb (1.13.0)
|
23
|
+
rdoc (>= 4.0.0)
|
24
|
+
reline (>= 0.4.2)
|
25
|
+
minitest (5.22.3)
|
26
|
+
multipart-post (2.4.0)
|
27
|
+
net-http (0.4.1)
|
28
|
+
uri
|
29
|
+
psych (5.1.2)
|
30
|
+
stringio
|
31
|
+
rake (13.2.1)
|
32
|
+
rdoc (6.6.3.1)
|
33
|
+
psych (>= 4.0.0)
|
34
|
+
reline (0.5.4)
|
35
|
+
io-console (~> 0.5)
|
36
|
+
ruby-openai (7.0.1)
|
37
|
+
event_stream_parser (>= 0.3.0, < 2.0.0)
|
38
|
+
faraday (>= 1)
|
39
|
+
faraday-multipart (>= 1)
|
40
|
+
stringio (3.1.0)
|
41
|
+
uri (0.13.0)
|
42
|
+
xxhash (0.5.0)
|
43
|
+
|
44
|
+
PLATFORMS
|
45
|
+
arm64-darwin-23
|
46
|
+
ruby
|
47
|
+
|
48
|
+
DEPENDENCIES
|
49
|
+
bundler (~> 2.5)
|
50
|
+
debug (~> 1.9)
|
51
|
+
minitest (~> 5.0)
|
52
|
+
minitest-smartdiff!
|
53
|
+
rake (~> 13.2)
|
54
|
+
|
55
|
+
BUNDLED WITH
|
56
|
+
2.5.3
|
data/README.md
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
# Minitest::Smartdiff
|
2
|
+
|
3
|
+
Have you ever found yourself staring cross-eyed and bewildered at walls of diffs that are **absolutely the same damn it**.
|
4
|
+
|
5
|
+
Well, it's the future, and you don't have to do that anymore. Let the endlessly patient Robot go looking for the needle in the needle stack.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'minitest-smartdiff'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install minitest-smartdiff
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
In your `test_helper.rb` file:
|
26
|
+
|
27
|
+
```ruby
|
28
|
+
require 'minitest-smartdiff'
|
29
|
+
```
|
30
|
+
|
31
|
+
And in your test file:
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
class MyTestThatProducesAnAnnoyingDiff < Minitest::Test
|
35
|
+
include Minitest::Smartdiff
|
36
|
+
|
37
|
+
def test_that_produces_an_annoying_obscure_diff
|
38
|
+
some_result = my_method()
|
39
|
+
|
40
|
+
smart_diff do
|
41
|
+
assert_equal "Mayonnaise is delicious", "Mayonnaise is delicioous"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
```
|
46
|
+
|
47
|
+
Alternatively - you can call `smart_diff_on` and `smart_diff_off` to turn the functionality on/off without using a block.
|
48
|
+
|
49
|
+
Smartdiff is smart enough not to ask the LLM for the difference if the assertion passes, so you don't accrue OpenAI costs for passing tests. However - you should consider this primarily a debugging tool - use at high volume at your own risk.
|
50
|
+
|
51
|
+
You can configure the OpenAI client like so:
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
class MyTestThatProducesAnAnnoyingDiff < Minitest::Test
|
55
|
+
include Minitest::Smartdiff
|
56
|
+
|
57
|
+
openai({
|
58
|
+
access_token: ENV['OPENAI_KEY']
|
59
|
+
})
|
60
|
+
|
61
|
+
model('gpt-3.5-turbo')
|
62
|
+
|
63
|
+
prompt(<<~ERB
|
64
|
+
You are a differ - you diff things. Diff this:
|
65
|
+
<%= expected %>
|
66
|
+
<%= actual %>
|
67
|
+
<%= mode %> is either `:json`, `:object` or `:text`
|
68
|
+
ERB
|
69
|
+
)
|
70
|
+
|
71
|
+
|
72
|
+
def test_that_produces_an_annoying_obscure_diff
|
73
|
+
some_result = my_method()
|
74
|
+
|
75
|
+
smart_diff do
|
76
|
+
assert_equal "Mayonnaise is delicious", "Mayonnaise is delicioous"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
```
|
81
|
+
|
82
|
+
## Development
|
83
|
+
|
84
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
85
|
+
|
86
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
87
|
+
|
88
|
+
## Contributing
|
89
|
+
|
90
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/stephenprater/minitest-smartdiff.
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "minitest/smartdiff"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
@@ -0,0 +1,177 @@
|
|
1
|
+
require "minitest/smartdiff/version"
|
2
|
+
require "minitest/assertions"
|
3
|
+
require "xxhash"
|
4
|
+
require "openai"
|
5
|
+
require "forwardable"
|
6
|
+
require "erb"
|
7
|
+
|
8
|
+
module Minitest
|
9
|
+
module Smartdiff
|
10
|
+
class Error < StandardError; end
|
11
|
+
|
12
|
+
extend Forwardable
|
13
|
+
|
14
|
+
def self.included(base)
|
15
|
+
base.extend ClassMethods
|
16
|
+
end
|
17
|
+
|
18
|
+
def smart_diff_off
|
19
|
+
@smart_diff = false
|
20
|
+
end
|
21
|
+
|
22
|
+
def smart_diff_on
|
23
|
+
@smart_diff = true
|
24
|
+
end
|
25
|
+
|
26
|
+
def smart_diff
|
27
|
+
raise ArgumentError, "Expected a block but none was given" unless block_given?
|
28
|
+
smart_diff_on
|
29
|
+
yield
|
30
|
+
ensure
|
31
|
+
smart_diff_off
|
32
|
+
end
|
33
|
+
|
34
|
+
def_delegators :"self.class", :openai, :prompt, :model, :valid_json?, :smart_diffable?
|
35
|
+
|
36
|
+
module ClassMethods
|
37
|
+
def openai(config = {})
|
38
|
+
@openai ||= OpenAI::Client.new(
|
39
|
+
{
|
40
|
+
organization_id: "",
|
41
|
+
log_errors: true,
|
42
|
+
access_token: ENV['OPENAI_KEY'],
|
43
|
+
uri_base: ENV['OPENAI_URI_BASE'],
|
44
|
+
}.merge(config)
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
def prompt(string = nil)
|
49
|
+
@prompt ||= string
|
50
|
+
@prompt ||= ERB.new <<~DEFAULT
|
51
|
+
You are Minitest::Smartdiff - the smartest differ that ever existed.
|
52
|
+
Your task is to find subtle but important differences in two pieces of data,
|
53
|
+
the expected, and the actual. When you find the difference describe the
|
54
|
+
difference as precisely as possible, and provide samples from the provided
|
55
|
+
input.
|
56
|
+
|
57
|
+
Be as succinct as possible, only point out the differences in the two strings.
|
58
|
+
|
59
|
+
Example Output
|
60
|
+
<% if mode == :json || mode == :object %>
|
61
|
+
Make sure to provide JSONPath to the location of the differences.
|
62
|
+
|
63
|
+
Expected:
|
64
|
+
{
|
65
|
+
usage: {
|
66
|
+
prompt_tokens: 9
|
67
|
+
}
|
68
|
+
}
|
69
|
+
|
70
|
+
Actual:
|
71
|
+
{
|
72
|
+
usage: {
|
73
|
+
prompt_tokens: 11
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
1. In the usage object, the number of tokens used is different:
|
78
|
+
- JSONPath: $.usage.prompt_tokens
|
79
|
+
- Expected: 9
|
80
|
+
- Actual: 11
|
81
|
+
<% else %>
|
82
|
+
Make sure to provide a concise description of the surrounding context for the differences.
|
83
|
+
|
84
|
+
Expected:
|
85
|
+
The quick brown dog jumped over the lazy red fox.
|
86
|
+
|
87
|
+
Actual:
|
88
|
+
The quick red dog jumped over the lazy brown fox.
|
89
|
+
|
90
|
+
1. The word "brown" in the expected text describes the dog, but in the actual text the word "red" is used.
|
91
|
+
2. The word "red" in the expected text describes the fox, is in the actual text the word "brown" is used.
|
92
|
+
<% end %>
|
93
|
+
|
94
|
+
This is very important to my career, I've got children to feed and a
|
95
|
+
mortgage to pay - so be very careful to only show differences.
|
96
|
+
|
97
|
+
Expected:
|
98
|
+
<%= expected %>
|
99
|
+
|
100
|
+
Actual
|
101
|
+
<%= actual %>
|
102
|
+
DEFAULT
|
103
|
+
end
|
104
|
+
|
105
|
+
def model(model = nil)
|
106
|
+
@model ||= model
|
107
|
+
@model ||= "gpt-3.5-turbo"
|
108
|
+
end
|
109
|
+
|
110
|
+
def valid_json?(str)
|
111
|
+
rep = JSON.parse(str)
|
112
|
+
rep.is_a?(Hash) || rep.is_a?(Array)
|
113
|
+
rescue => e
|
114
|
+
false
|
115
|
+
end
|
116
|
+
|
117
|
+
def smart_diffable?(exp, act)
|
118
|
+
return false if exp.class != act.class
|
119
|
+
return false unless [Hash, String, Array].include?(exp.class)
|
120
|
+
|
121
|
+
if exp.is_a?(String)
|
122
|
+
if valid_json?(exp) && valid_json?(act)
|
123
|
+
:json
|
124
|
+
else
|
125
|
+
:text
|
126
|
+
end
|
127
|
+
else
|
128
|
+
:object
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
module Assertions
|
135
|
+
alias_method :old_diff, :diff
|
136
|
+
|
137
|
+
def smart_diff(exp, act)
|
138
|
+
return old_diff(exp,act) unless @smart_diff
|
139
|
+
|
140
|
+
mode = smart_diffable?(exp, act)
|
141
|
+
|
142
|
+
return old_diff(exp, act) unless mode
|
143
|
+
|
144
|
+
expected, actual = if mode == :object
|
145
|
+
[exp.to_json, act.to_json]
|
146
|
+
else
|
147
|
+
[exp, act]
|
148
|
+
end
|
149
|
+
|
150
|
+
params = {
|
151
|
+
parameters: {
|
152
|
+
messages: [
|
153
|
+
{
|
154
|
+
role: "user",
|
155
|
+
content: prompt.result_with_hash({
|
156
|
+
expected:,
|
157
|
+
actual:,
|
158
|
+
mode:
|
159
|
+
}),
|
160
|
+
},
|
161
|
+
],
|
162
|
+
model: model,
|
163
|
+
temperature: 0.00000000000001,
|
164
|
+
seed: XXhash.xxh32(expected, actual)
|
165
|
+
}
|
166
|
+
}
|
167
|
+
|
168
|
+
completion = openai.chat(**params)
|
169
|
+
completion.dig("choices",0,"message","content") + "\n"
|
170
|
+
rescue => e
|
171
|
+
warn "Error talking to OpenAI: #{e.message} will fallback to basic diff"
|
172
|
+
old_diff(exp,act)
|
173
|
+
end
|
174
|
+
|
175
|
+
alias_method :diff, :smart_diff
|
176
|
+
end
|
177
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "minitest/smartdiff/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "minitest-smartdiff"
|
8
|
+
spec.version = Minitest::Smartdiff::VERSION
|
9
|
+
spec.authors = ["Stephen Prater", "Jeremy Cobb"]
|
10
|
+
spec.email = ["me@stephenprater.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Diffs are hard, make the robot do it.}
|
13
|
+
spec.description = %q{Diff between expected and actual with GPT}
|
14
|
+
spec.homepage = "https://github.com/stephenprater/minitest-smartdiff"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Specify which files should be added to the gem when it is released.
|
18
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
19
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
20
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
21
|
+
end
|
22
|
+
spec.bindir = "exe"
|
23
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
|
+
spec.require_paths = ["lib"]
|
25
|
+
|
26
|
+
spec.add_dependency "ruby-openai", "~> 7.0"
|
27
|
+
spec.add_dependency "xxhash", "~> 0.5"
|
28
|
+
|
29
|
+
spec.add_development_dependency "bundler", "~> 2.5"
|
30
|
+
spec.add_development_dependency "rake", "~> 13.2"
|
31
|
+
spec.add_development_dependency "minitest", "~> 5.0"
|
32
|
+
spec.add_development_dependency "debug", "~> 1.9"
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: minitest-smartdiff
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Stephen Prater
|
8
|
+
- Jeremy Cobb
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2024-05-02 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: ruby-openai
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '7.0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '7.0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: xxhash
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0.5'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0.5'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: bundler
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '2.5'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '2.5'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rake
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '13.2'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '13.2'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: minitest
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '5.0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '5.0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: debug
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '1.9'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '1.9'
|
98
|
+
description: Diff between expected and actual with GPT
|
99
|
+
email:
|
100
|
+
- me@stephenprater.com
|
101
|
+
executables: []
|
102
|
+
extensions: []
|
103
|
+
extra_rdoc_files: []
|
104
|
+
files:
|
105
|
+
- ".gitignore"
|
106
|
+
- ".travis.yml"
|
107
|
+
- Gemfile
|
108
|
+
- Gemfile.lock
|
109
|
+
- README.md
|
110
|
+
- Rakefile
|
111
|
+
- bin/console
|
112
|
+
- bin/setup
|
113
|
+
- lib/minitest/smartdiff.rb
|
114
|
+
- lib/minitest/smartdiff/version.rb
|
115
|
+
- minitest-smartdiff.gemspec
|
116
|
+
homepage: https://github.com/stephenprater/minitest-smartdiff
|
117
|
+
licenses:
|
118
|
+
- MIT
|
119
|
+
metadata: {}
|
120
|
+
post_install_message:
|
121
|
+
rdoc_options: []
|
122
|
+
require_paths:
|
123
|
+
- lib
|
124
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
125
|
+
requirements:
|
126
|
+
- - ">="
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '0'
|
129
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
130
|
+
requirements:
|
131
|
+
- - ">="
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
requirements: []
|
135
|
+
rubygems_version: 3.5.3
|
136
|
+
signing_key:
|
137
|
+
specification_version: 4
|
138
|
+
summary: Diffs are hard, make the robot do it.
|
139
|
+
test_files: []
|