hegeltest 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +262 -0
- data/README.ja.md +408 -0
- data/README.md +209 -103
- data/Rakefile +6 -3
- data/docs/README.md +11 -0
- data/docs/adr/0025-copy-each-string-input-into-a-hegel-string-t.md +61 -0
- data/docs/adr/0026-select-the-engine-release-repository-by-version.md +47 -0
- data/docs/adr/0027-report-each-runs-test-location-from-the-callers-frame.md +63 -0
- data/docs/adr/0028-expose-settings-profiles-as-a-keyword-and-module-functions.md +91 -0
- data/docs/adr/0029-keep-notes-local-and-defer-run-error-kinds.md +72 -0
- data/docs/adr/0030-represent-a-duration-as-integer-nanoseconds.md +48 -0
- data/docs/adr/0031-take-the-alphabet-options-from-hegel-rust.md +52 -0
- data/docs/adr/0032-suppress-span-closes-while-an-abandoned-recursion-attempt-unwinds.md +147 -0
- data/docs/adr/0033-cache-compiled-grammars-for-the-length-of-a-run.md +80 -0
- data/docs/adr/0034-repeat-a-test-body-with-a-ruby-side-indent.md +120 -0
- data/docs/adr/0035-defer-concurrent-state-machines-and-the-engine-printer.md +136 -0
- data/docs/architecture.md +33 -11
- data/lib/hegel/errors.rb +24 -0
- data/lib/hegel/generator.rb +52 -25
- data/lib/hegel/generators.rb +715 -45
- data/lib/hegel/grammar_cache.rb +36 -0
- data/lib/hegel/lib_hegel/real.rb +455 -120
- data/lib/hegel/lib_hegel.rb +33 -8
- data/lib/hegel/libhegel_version.rb +1 -1
- data/lib/hegel/locate.rb +10 -10
- data/lib/hegel/report.rb +13 -10
- data/lib/hegel/runner.rb +61 -27
- data/lib/hegel/settings.rb +92 -16
- data/lib/hegel/state_machine.rb +17 -4
- data/lib/hegel/stateful/pool.rb +52 -0
- data/lib/hegel/stateful.rb +2 -2
- data/lib/hegel/syntax/methods.rb +99 -15
- data/lib/hegel/test_case.rb +256 -35
- data/lib/hegel/test_location.rb +49 -0
- data/lib/hegel/version.rb +1 -1
- data/lib/hegel.rb +21 -5
- data/lib/tasks/libhegel.rake +17 -5
- data/lib/tasks/platform_gems.rake +3 -3
- data/sig/hegel.rbs +296 -43
- data/skills/hegel-ruby/references/ruby/reference.md +465 -43
- metadata +15 -1
data/lib/hegel/lib_hegel.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Hegel
|
|
|
18
18
|
# is testable without the native engine. hegel-java makes the same split,
|
|
19
19
|
# between its Libhegel interface, RealLibhegel, and FakeLibhegel.
|
|
20
20
|
module LibHegel
|
|
21
|
-
# hegel_result_t, named from
|
|
21
|
+
# hegel_result_t, named from libhegel/include/hegel.h's enum of the same
|
|
22
22
|
# name. HEGEL_OK is success; every other value is a negative error code.
|
|
23
23
|
HEGEL_OK = 0
|
|
24
24
|
HEGEL_E_STOP_TEST = -1
|
|
@@ -30,6 +30,8 @@ module Hegel
|
|
|
30
30
|
HEGEL_E_NOT_COMPLETE = -7
|
|
31
31
|
HEGEL_E_INTERNAL = -8
|
|
32
32
|
HEGEL_E_CONCURRENT_USE = -9
|
|
33
|
+
HEGEL_E_RETRY = -10
|
|
34
|
+
HEGEL_E_NOT_FOUND = -11
|
|
33
35
|
|
|
34
36
|
# The hegel.h name for each code above, so a translated error message
|
|
35
37
|
# names the code instead of leaving the reader to cross-reference the
|
|
@@ -44,7 +46,9 @@ module Hegel
|
|
|
44
46
|
HEGEL_E_ALREADY_COMPLETE => "HEGEL_E_ALREADY_COMPLETE",
|
|
45
47
|
HEGEL_E_NOT_COMPLETE => "HEGEL_E_NOT_COMPLETE",
|
|
46
48
|
HEGEL_E_INTERNAL => "HEGEL_E_INTERNAL",
|
|
47
|
-
HEGEL_E_CONCURRENT_USE => "HEGEL_E_CONCURRENT_USE"
|
|
49
|
+
HEGEL_E_CONCURRENT_USE => "HEGEL_E_CONCURRENT_USE",
|
|
50
|
+
HEGEL_E_RETRY => "HEGEL_E_RETRY",
|
|
51
|
+
HEGEL_E_NOT_FOUND => "HEGEL_E_NOT_FOUND"
|
|
48
52
|
}.freeze
|
|
49
53
|
|
|
50
54
|
# hegel_status_t, named from hegel.h's enum of the same name. Passed to
|
|
@@ -67,6 +71,15 @@ module Hegel
|
|
|
67
71
|
HEGEL_VERBOSITY_VERBOSE = 2
|
|
68
72
|
HEGEL_VERBOSITY_DEBUG = 3
|
|
69
73
|
|
|
74
|
+
# hegel_nondeterminism_strictness_t
|
|
75
|
+
HEGEL_NONDETERMINISM_QUIET = 0
|
|
76
|
+
HEGEL_NONDETERMINISM_WARN = 1
|
|
77
|
+
HEGEL_NONDETERMINISM_ERROR = 2
|
|
78
|
+
|
|
79
|
+
# hegel_backend_t
|
|
80
|
+
HEGEL_BACKEND_DEFAULT = 1
|
|
81
|
+
HEGEL_BACKEND_URANDOM = 2
|
|
82
|
+
|
|
70
83
|
# hegel_phase_t, named from hegel.h's enum of the same name. A bitwise
|
|
71
84
|
# OR of these is passed to hegel_settings_set_phases; the default is
|
|
72
85
|
# HEGEL_PHASE_ALL.
|
|
@@ -108,7 +121,8 @@ module Hegel
|
|
|
108
121
|
# responds to each name here.
|
|
109
122
|
METHODS = %i[
|
|
110
123
|
context_new context_free context_last_error version
|
|
111
|
-
settings_new
|
|
124
|
+
settings_new settings_new_for_profile settings_register_profile set_default_profile
|
|
125
|
+
settings_free settings_set_test_cases settings_set_verbosity
|
|
112
126
|
settings_set_seed settings_set_derandomize settings_set_database
|
|
113
127
|
run_start next_test_case run_free test_case_free mark_complete
|
|
114
128
|
generate_boolean generate_integer generate_integer_big
|
|
@@ -118,19 +132,30 @@ module Hegel
|
|
|
118
132
|
start_span stop_span
|
|
119
133
|
new_collection collection_more collection_reject collection_free
|
|
120
134
|
generate_float
|
|
121
|
-
string_generator_text string_generator_free generate_string
|
|
135
|
+
string_generator_text string_generator_free generate_string owned_string_free
|
|
122
136
|
generate_bytes generate_bytes_result_free
|
|
123
137
|
string_generator_regex string_generator_email string_generator_url string_generator_domain
|
|
124
138
|
generate_ipv4 generate_ipv6 generate_uuid
|
|
125
139
|
generate_date generate_time generate_datetime
|
|
126
140
|
settings_set_phases settings_set_suppress_health_check settings_set_report_multiple_failures
|
|
127
|
-
settings_set_database_key
|
|
128
|
-
|
|
129
|
-
|
|
141
|
+
settings_set_database_key settings_set_stop_on_first_failure settings_set_print_blob
|
|
142
|
+
settings_get_print_blob settings_set_show_statistics settings_set_nondeterminism_strictness
|
|
143
|
+
settings_set_backend settings_set_test_location
|
|
144
|
+
target event event_value
|
|
145
|
+
new_pool pool_add pool_generate pool_restore pool_free
|
|
130
146
|
new_state_machine state_machine_next_group state_machine_next_rule state_machine_rule_rejected
|
|
131
147
|
state_machine_should_check_invariant state_machine_free
|
|
148
|
+
grammar_builder_new grammar_builder_file grammar_builder_provide grammar_builder_free
|
|
149
|
+
grammar_new grammar_placeholder grammar_free generate_grammar
|
|
150
|
+
new_recursion recursion_branch recursion_leaf recursion_retry recursion_finish recursion_free
|
|
132
151
|
].freeze
|
|
133
152
|
|
|
153
|
+
# label_combine answers from this table: a recursive generator builds a
|
|
154
|
+
# fresh branch generator at every node it draws, and each one combines
|
|
155
|
+
# the same few labels again. Measured on the 30-leaf shrink in
|
|
156
|
+
# test_recursive_engine.rb, hashing the labels was a third of the run.
|
|
157
|
+
COMBINED_LABELS = {}
|
|
158
|
+
|
|
134
159
|
module_function
|
|
135
160
|
|
|
136
161
|
# Runs the block with a context obtained from +impl.context_new+,
|
|
@@ -200,7 +225,7 @@ module Hegel
|
|
|
200
225
|
end
|
|
201
226
|
|
|
202
227
|
def label_combine(labels)
|
|
203
|
-
fnv1a(labels.pack("Q<*").bytes)
|
|
228
|
+
COMBINED_LABELS[labels] ||= fnv1a(labels.pack("Q<*").bytes)
|
|
204
229
|
end
|
|
205
230
|
|
|
206
231
|
def fnv1a(bytes)
|
|
@@ -5,5 +5,5 @@ module Hegel
|
|
|
5
5
|
# Hegel::VERSION (the gem's own release): hegel-go and hegel-typescript keep
|
|
6
6
|
# the two separate too, so bumping the pinned engine can land as its own
|
|
7
7
|
# commit without also releasing a new gem version.
|
|
8
|
-
LIBHEGEL_VERSION = "0.
|
|
8
|
+
LIBHEGEL_VERSION = "0.48.0"
|
|
9
9
|
end
|
data/lib/hegel/locate.rb
CHANGED
|
@@ -16,7 +16,7 @@ module Hegel
|
|
|
16
16
|
# fallback.
|
|
17
17
|
# 2. The copy bundled into the gem under +lib/hegel/libhegel/+.
|
|
18
18
|
#
|
|
19
|
-
# A sibling `../hegel
|
|
19
|
+
# A sibling engine checkout (`../hegel`) is deliberately not searched. The env
|
|
20
20
|
# override already covers local-engine-build workflows without depending
|
|
21
21
|
# on any particular checkout layout, and `rake libhegel:fetch` covers the
|
|
22
22
|
# rest; a rule that depends on a layout this repo's own checkout does not
|
|
@@ -32,10 +32,10 @@ module Hegel
|
|
|
32
32
|
LIBRARY_PATH_ENV = "HEGEL_LIBHEGEL_PATH"
|
|
33
33
|
|
|
34
34
|
# Published release asset name for each supported "<host_cpu>-<host_os>"
|
|
35
|
-
# pair, mirroring the artifacts
|
|
36
|
-
# LIBHEGEL_VERSION. Hosts absent from this table (e.g.
|
|
37
|
-
# Intel Mac) are unsupported because
|
|
38
|
-
# for them.
|
|
35
|
+
# pair, mirroring the artifacts the libhegel release of the pinned
|
|
36
|
+
# LIBHEGEL_VERSION publishes. Hosts absent from this table (e.g.
|
|
37
|
+
# x86_64-darwin, Intel Mac) are unsupported because that release does not
|
|
38
|
+
# publish a build for them.
|
|
39
39
|
ASSET_NAMES = {
|
|
40
40
|
"arm64-darwin" => "libhegel-darwin-arm64.dylib",
|
|
41
41
|
"x86_64-linux" => "libhegel-linux-amd64.so",
|
|
@@ -76,7 +76,7 @@ module Hegel
|
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
# The release asset name for the given host, or raises Hegel::Error if
|
|
79
|
-
#
|
|
79
|
+
# the libhegel release does not publish a build for it.
|
|
80
80
|
def asset_name(host_cpu:, host_os:)
|
|
81
81
|
release_asset_name(host_cpu, host_os) || raise(Hegel::Error, unsupported_host_message(host_cpu, host_os))
|
|
82
82
|
end
|
|
@@ -101,7 +101,7 @@ module Hegel
|
|
|
101
101
|
# directory override is how an unsupported host (e.g. x86_64-darwin)
|
|
102
102
|
# points at a local build in the first place, so treating it as an
|
|
103
103
|
# error here would close off that path.
|
|
104
|
-
# 2. libhegel_c.<ext>, the name cargo gives the library when
|
|
104
|
+
# 2. libhegel_c.<ext>, the name cargo gives the library when libhegel's
|
|
105
105
|
# crate (`[lib] name = "hegel_c"`) is built without renaming it.
|
|
106
106
|
# 3. libhegel.<ext>, a renamed copy, per the hegel-go and hegel-ocaml
|
|
107
107
|
# READMEs.
|
|
@@ -122,8 +122,8 @@ module Hegel
|
|
|
122
122
|
[release_asset_name(host_cpu, host_os), "libhegel_c.#{ext}", "libhegel.#{ext}"].compact
|
|
123
123
|
end
|
|
124
124
|
|
|
125
|
-
# The release asset name for the given host, or nil if
|
|
126
|
-
# not publish a build for it. Split from asset_name (which raises) so a
|
|
125
|
+
# The release asset name for the given host, or nil if the libhegel
|
|
126
|
+
# release does not publish a build for it. Split from asset_name (which raises) so a
|
|
127
127
|
# directory override can treat an unsupported host as "skip this
|
|
128
128
|
# candidate" instead of an error.
|
|
129
129
|
def release_asset_name(host_cpu, host_os)
|
|
@@ -159,7 +159,7 @@ module Hegel
|
|
|
159
159
|
end
|
|
160
160
|
|
|
161
161
|
# Shared-library extension for host_os alone, independent of whether the
|
|
162
|
-
# specific host_cpu is one
|
|
162
|
+
# specific host_cpu is one libhegel publishes a build for. This lets
|
|
163
163
|
# an explicit directory override (step 1) resolve on an otherwise
|
|
164
164
|
# unsupported host (e.g. x86_64-darwin), matching a local libhegel build
|
|
165
165
|
# that always produces the OS-native extension.
|
data/lib/hegel/report.rb
CHANGED
|
@@ -13,9 +13,10 @@ module Hegel
|
|
|
13
13
|
# One failure's rendered ingredients. +test_cases+ and +discarded+ are
|
|
14
14
|
# the generation phase's own counts up to that failure's first
|
|
15
15
|
# appearance (see Hegel::Runner::GenerationStats), not the shrink
|
|
16
|
-
# phase's. +entries+ is the [:draw, name, value
|
|
17
|
-
# Hegel::TestCase recorded on the stamped
|
|
18
|
-
# failure, in call order, still
|
|
16
|
+
# phase's. +entries+ is the [:draw, name, value, indent] /
|
|
17
|
+
# [:note, message, indent] list Hegel::TestCase recorded on the stamped
|
|
18
|
+
# case that produced this failure, in call order, still
|
|
19
|
+
# un-#inspect'd/un-#to_s'd (see
|
|
19
20
|
# Hegel::TestCase#record_draw and #note for why). +blob+ is the string
|
|
20
21
|
# Hegel.test(reproduce_failure:) accepts to replay this same failure, or
|
|
21
22
|
# nil when the engine produced none. +caveat+ is the engine's own account
|
|
@@ -41,10 +42,10 @@ module Hegel
|
|
|
41
42
|
entries.map do |entry|
|
|
42
43
|
next entry unless entry[0] == :draw
|
|
43
44
|
|
|
44
|
-
_tag, name, value = entry
|
|
45
|
+
_tag, name, value, indent = entry
|
|
45
46
|
if counts[name] > 1
|
|
46
47
|
seen[name] += 1
|
|
47
|
-
[:draw, "#{name}_#{seen[name]}", value]
|
|
48
|
+
[:draw, "#{name}_#{seen[name]}", value, indent]
|
|
48
49
|
else
|
|
49
50
|
entry
|
|
50
51
|
end
|
|
@@ -60,17 +61,19 @@ module Hegel
|
|
|
60
61
|
# the same. A :note shares the :draw lines' 2-space indent
|
|
61
62
|
# deliberately: both belong to the same block, and a different indent
|
|
62
63
|
# would read as a different kind of thing instead of the same
|
|
63
|
-
# call-order list.
|
|
64
|
+
# call-order list. Each entry's own indent goes on top of those 2
|
|
65
|
+
# columns, which is how an iteration of Hegel::TestCase#repeat sits
|
|
66
|
+
# under its "// Repetition #N" line.
|
|
64
67
|
def render_failure(failure)
|
|
65
68
|
cases = "#{failure.test_cases} test #{(failure.test_cases == 1) ? "case" : "cases"}"
|
|
66
69
|
lines = ["Falsified after #{cases} (#{failure.discarded} discarded):", ""]
|
|
67
70
|
assign_names(failure.entries).each do |entry|
|
|
68
71
|
if entry[0] == :draw
|
|
69
|
-
_tag, name, value = entry
|
|
70
|
-
lines << " #{name} = #{value.inspect}"
|
|
72
|
+
_tag, name, value, indent = entry
|
|
73
|
+
lines << " #{" " * indent}#{name} = #{value.inspect}"
|
|
71
74
|
else
|
|
72
|
-
_tag, message = entry
|
|
73
|
-
lines << " #{message}"
|
|
75
|
+
_tag, message, indent = entry
|
|
76
|
+
lines << " #{" " * indent}#{message}"
|
|
74
77
|
end
|
|
75
78
|
end
|
|
76
79
|
lines.push("", "note: #{failure.caveat}") if failure.caveat
|
data/lib/hegel/runner.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
|
+
require_relative "grammar_cache"
|
|
4
5
|
require_relative "lib_hegel"
|
|
5
6
|
require_relative "report"
|
|
6
7
|
require_relative "settings"
|
|
@@ -97,10 +98,8 @@ module Hegel
|
|
|
97
98
|
|
|
98
99
|
module_function
|
|
99
100
|
|
|
100
|
-
# Runs +block+ as a Hegel property against +impl+, applying
|
|
101
|
-
#
|
|
102
|
-
# +phases+, +suppress_health_check+, and +report_multiple_failures+ (see
|
|
103
|
-
# Hegel::Settings) to a fresh settings handle first. Returns nil on a
|
|
101
|
+
# Runs +block+ as a Hegel property against +impl+, applying the
|
|
102
|
+
# settings keywords (see Hegel::Settings) to a fresh settings handle first. Returns nil on a
|
|
104
103
|
# passing run, re-raises the exception the smallest failing case's body
|
|
105
104
|
# raised on a failing run, and raises Hegel::Error for a run-level
|
|
106
105
|
# failure (ERROR status, a blob that no longer reproduces, or more than
|
|
@@ -119,6 +118,13 @@ module Hegel
|
|
|
119
118
|
# with its class and backtrace intact), so a hegel.toml profile or a
|
|
120
119
|
# HEGEL_* variable does not get to turn it off by accident.
|
|
121
120
|
#
|
|
121
|
+
# +profile+ names the engine profile the settings start from; nil starts
|
|
122
|
+
# from the default profile. The keywords apply on top either way, so one
|
|
123
|
+
# passed explicitly beats the profile's value.
|
|
124
|
+
#
|
|
125
|
+
# +test_location+ is a Hegel::TestLocation. It is identity rather than a
|
|
126
|
+
# setting, so it goes to the engine here and not through Settings.apply.
|
|
127
|
+
#
|
|
122
128
|
# +reproduce_failure+, when given, starts the run from that blob instead
|
|
123
129
|
# (see LibHegel::Real#run_start_blob): the engine replays it rather than
|
|
124
130
|
# exploring, and the same loop below drives the replay.
|
|
@@ -130,16 +136,27 @@ module Hegel
|
|
|
130
136
|
# `ensure` by the code that opened it, innermost first: hegel_context_free
|
|
131
137
|
# requires every other handle taken from the context to be freed first,
|
|
132
138
|
# and Ruby's GC gives finalizers no ordering guarantee to rely on instead.
|
|
133
|
-
|
|
134
|
-
|
|
139
|
+
#
|
|
140
|
+
# The grammars the run compiled are freed after the run and before the
|
|
141
|
+
# context (docs/adr/0033).
|
|
142
|
+
def run(impl:, test_location:, profile: nil, test_cases: nil, seed: nil, derandomize: nil, verbosity: nil, database: nil, database_key: nil,
|
|
143
|
+
phases: nil, suppress_health_check: nil, report_multiple_failures: false, stop_on_first_failure: nil,
|
|
144
|
+
print_blob: nil, show_statistics: nil, nondeterminism_strictness: nil, backend: nil,
|
|
135
145
|
output: $stderr, reproduce_failure: nil, &block)
|
|
136
146
|
quiet = verbosity == :quiet
|
|
137
147
|
LibHegel.with_context(impl) do |ctx|
|
|
138
|
-
settings =
|
|
148
|
+
settings = Settings.new_handle(impl, ctx, profile)
|
|
139
149
|
run = begin
|
|
140
150
|
Settings.apply(impl, ctx, settings, test_cases: test_cases, seed: seed, derandomize: derandomize,
|
|
141
151
|
verbosity: verbosity, database: database, database_key: database_key, phases: phases,
|
|
142
|
-
suppress_health_check: suppress_health_check, report_multiple_failures: report_multiple_failures
|
|
152
|
+
suppress_health_check: suppress_health_check, report_multiple_failures: report_multiple_failures,
|
|
153
|
+
stop_on_first_failure: stop_on_first_failure, print_blob: print_blob, show_statistics: show_statistics,
|
|
154
|
+
nondeterminism_strictness: nondeterminism_strictness, backend: backend)
|
|
155
|
+
impl.settings_set_test_location(ctx, settings, test_location.file, test_location.begin_line,
|
|
156
|
+
test_location.class_name, test_location.function)
|
|
157
|
+
# Read back rather than taken from the keyword, so that a profile
|
|
158
|
+
# or HEGEL_PRINT_BLOB decides when the keyword is nil.
|
|
159
|
+
print_blob = impl.settings_get_print_blob(ctx, settings)
|
|
143
160
|
if reproduce_failure
|
|
144
161
|
impl.run_start_blob(ctx, settings, reproduce_failure)
|
|
145
162
|
else
|
|
@@ -150,19 +167,26 @@ module Hegel
|
|
|
150
167
|
# hegel_run_start copies them.
|
|
151
168
|
impl.settings_free(ctx, settings)
|
|
152
169
|
end
|
|
153
|
-
|
|
170
|
+
grammars = GrammarCache.new
|
|
171
|
+
begin
|
|
172
|
+
run_and_finish(impl, ctx, run, grammars, reproducing: !reproduce_failure.nil?, quiet: quiet,
|
|
173
|
+
print_blob: print_blob, output: output, &block)
|
|
174
|
+
ensure
|
|
175
|
+
grammars.free_all(impl, ctx)
|
|
176
|
+
end
|
|
154
177
|
end
|
|
155
178
|
end
|
|
156
179
|
|
|
157
180
|
# Drives +run+ and hands its result to #finish.
|
|
158
|
-
def run_and_finish(impl, ctx, run, reproducing:, quiet:, output:, &block)
|
|
181
|
+
def run_and_finish(impl, ctx, run, grammars, reproducing:, quiet:, print_blob:, output:, &block)
|
|
159
182
|
stats = GenerationStats.new
|
|
160
183
|
captures = {}
|
|
161
|
-
drive(impl, ctx, run, stats, captures, &block)
|
|
184
|
+
drive(impl, ctx, run, grammars, stats, captures, &block)
|
|
162
185
|
|
|
163
186
|
result = impl.run_result(ctx, run)
|
|
164
187
|
begin
|
|
165
|
-
finish(impl, ctx, result, stats, captures, reproducing: reproducing, quiet: quiet,
|
|
188
|
+
finish(impl, ctx, result, stats, captures, reproducing: reproducing, quiet: quiet, print_blob: print_blob,
|
|
189
|
+
output: output)
|
|
166
190
|
ensure
|
|
167
191
|
impl.run_result_free(ctx, result)
|
|
168
192
|
end
|
|
@@ -178,16 +202,16 @@ module Hegel
|
|
|
178
202
|
# Pulls test cases from +run+ until hegel_next_test_case reports none
|
|
179
203
|
# left (a nil out-parameter, not an error). Never counts iterations
|
|
180
204
|
# itself: test_cases bounds generation, not how many times shrinking
|
|
181
|
-
# and rejected cases call the body. Measured against libhegel 0.
|
|
205
|
+
# and rejected cases call the body. Measured against libhegel 0.48.0, a
|
|
182
206
|
# run configured for 20 test cases whose body rejected nearly every case
|
|
183
|
-
# took
|
|
207
|
+
# took 782 to 2047 iterations over 30 runs. +stats+ does its own, different counting
|
|
184
208
|
# -- see GenerationStats above.
|
|
185
|
-
def drive(impl, ctx, run, stats, captures, &block)
|
|
209
|
+
def drive(impl, ctx, run, grammars, stats, captures, &block)
|
|
186
210
|
loop do
|
|
187
211
|
tc = impl.next_test_case(ctx, run)
|
|
188
212
|
break if tc.nil?
|
|
189
213
|
|
|
190
|
-
run_case(impl, ctx, tc, stats, captures, &block)
|
|
214
|
+
run_case(impl, ctx, tc, grammars, stats, captures, &block)
|
|
191
215
|
end
|
|
192
216
|
end
|
|
193
217
|
|
|
@@ -207,8 +231,8 @@ module Hegel
|
|
|
207
231
|
# The case records its draws only when the engine stamped it for
|
|
208
232
|
# capture. The stamp is read in here, inside the `ensure`, so the handle
|
|
209
233
|
# is freed even when reading it raises.
|
|
210
|
-
def with_test_case(impl, ctx, tc)
|
|
211
|
-
test_case = TestCase.new(impl, ctx, tc, record: impl.test_case_should_capture(ctx, tc))
|
|
234
|
+
def with_test_case(impl, ctx, tc, grammars)
|
|
235
|
+
test_case = TestCase.new(impl, ctx, tc, record: impl.test_case_should_capture(ctx, tc), grammars: grammars)
|
|
212
236
|
yield test_case
|
|
213
237
|
ensure
|
|
214
238
|
begin
|
|
@@ -228,8 +252,8 @@ module Hegel
|
|
|
228
252
|
# and still reaches #with_test_case's own `ensure`, so the handle is
|
|
229
253
|
# freed either way; its owner is this loop, not hegel_run_free (see
|
|
230
254
|
# #run_and_finish's comment above).
|
|
231
|
-
def run_case(impl, ctx, tc, stats, captures, &block)
|
|
232
|
-
with_test_case(impl, ctx, tc) do |test_case|
|
|
255
|
+
def run_case(impl, ctx, tc, grammars, stats, captures, &block)
|
|
256
|
+
with_test_case(impl, ctx, tc, grammars) do |test_case|
|
|
233
257
|
status, origin, exception, entries = classify(test_case, &block)
|
|
234
258
|
stats.record(status, origin)
|
|
235
259
|
keep_capture(captures, origin, exception, entries) if status == LibHegel::HEGEL_STATUS_INTERESTING
|
|
@@ -258,7 +282,7 @@ module Hegel
|
|
|
258
282
|
# would mean this binding does not recognise a hegel_run_status_t value
|
|
259
283
|
# the loaded engine returned, mirroring how LibHegel.check! names an
|
|
260
284
|
# unrecognised result code instead of silently doing nothing with it.
|
|
261
|
-
def finish(impl, ctx, result, stats, captures, reproducing:, quiet:, output:)
|
|
285
|
+
def finish(impl, ctx, result, stats, captures, reproducing:, quiet:, print_blob:, output:)
|
|
262
286
|
status = impl.run_result_status(ctx, result)
|
|
263
287
|
case status
|
|
264
288
|
when LibHegel::HEGEL_RUN_STATUS_PASSED
|
|
@@ -266,7 +290,7 @@ module Hegel
|
|
|
266
290
|
when LibHegel::HEGEL_RUN_STATUS_ERROR
|
|
267
291
|
raise Hegel::Error, impl.run_result_error(ctx, result) || UNKNOWN_RUN_ERROR_MESSAGE
|
|
268
292
|
when LibHegel::HEGEL_RUN_STATUS_FAILED
|
|
269
|
-
report_failures(impl, ctx, result, stats, captures, quiet: quiet, output: output)
|
|
293
|
+
report_failures(impl, ctx, result, stats, captures, quiet: quiet, print_blob: print_blob, output: output)
|
|
270
294
|
else
|
|
271
295
|
raise Hegel::Error, "hegel: run finished with an unrecognized status (#{status})"
|
|
272
296
|
end
|
|
@@ -286,13 +310,13 @@ module Hegel
|
|
|
286
310
|
# guard. The report is written before either raise, not after: a host
|
|
287
311
|
# framework that catches the re-raised exception would otherwise read
|
|
288
312
|
# its own output before this one, out of order.
|
|
289
|
-
def report_failures(impl, ctx, result, stats, captures, quiet:, output:)
|
|
313
|
+
def report_failures(impl, ctx, result, stats, captures, quiet:, print_blob:, output:)
|
|
290
314
|
exceptions = []
|
|
291
315
|
failures = []
|
|
292
316
|
impl.run_result_failure_count(ctx, result).times do |index|
|
|
293
317
|
failure = impl.run_result_failure(ctx, result, index)
|
|
294
318
|
begin
|
|
295
|
-
exception, report = failure_report(impl, ctx, failure, stats, captures)
|
|
319
|
+
exception, report = failure_report(impl, ctx, failure, stats, captures, print_blob: print_blob)
|
|
296
320
|
exceptions << exception
|
|
297
321
|
failures << report
|
|
298
322
|
ensure
|
|
@@ -310,13 +334,13 @@ module Hegel
|
|
|
310
334
|
# case the run produced, so every failure has a capture under its origin.
|
|
311
335
|
# A capture with no entries came from an unstamped case, whose draws
|
|
312
336
|
# went unrecorded, so its report lists no values.
|
|
313
|
-
def failure_report(impl, ctx, failure, stats, captures)
|
|
337
|
+
def failure_report(impl, ctx, failure, stats, captures, print_blob:)
|
|
314
338
|
origin = impl.failure_origin(ctx, failure)
|
|
315
339
|
capture = captures.fetch(origin)
|
|
316
340
|
test_cases, discarded = stats.for(origin)
|
|
317
341
|
report = Report::Failure.new(
|
|
318
342
|
test_cases: test_cases, discarded: discarded, entries: capture.entries || [],
|
|
319
|
-
blob: impl.failure_reproduction_blob(ctx, failure), caveat: impl.failure_caveat(ctx, failure)
|
|
343
|
+
blob: (impl.failure_reproduction_blob(ctx, failure) if print_blob), caveat: impl.failure_caveat(ctx, failure)
|
|
320
344
|
)
|
|
321
345
|
[capture.exception, report]
|
|
322
346
|
end
|
|
@@ -344,10 +368,20 @@ module Hegel
|
|
|
344
368
|
[LibHegel::HEGEL_STATUS_VALID, nil, nil, nil]
|
|
345
369
|
rescue *Hegel::FATAL_EXCEPTIONS
|
|
346
370
|
raise
|
|
347
|
-
|
|
371
|
+
# A LeafBudgetExceeded gets here only when a body swallowed the
|
|
372
|
+
# exception that replaced it mid-unwind and then drew again. It is
|
|
373
|
+
# library control flow, not a property failing, and its attempt never
|
|
374
|
+
# reached hegel_recursion_retry, so the case is discarded.
|
|
375
|
+
rescue Hegel::AssumeFailed, Hegel::LeafBudgetExceeded
|
|
348
376
|
[LibHegel::HEGEL_STATUS_INVALID, nil, nil, nil]
|
|
349
377
|
rescue Hegel::StopTest
|
|
350
378
|
[LibHegel::HEGEL_STATUS_OVERRUN, nil, nil, nil]
|
|
379
|
+
rescue Hegel::LoopDone
|
|
380
|
+
# Hegel::TestCase#repeat ran until the engine stopped it, with no
|
|
381
|
+
# failure; hegel-rust's run lifecycle maps its LoopDone to VALID too.
|
|
382
|
+
# .mutineer.yml ignores the mutants of its last two elements by id, for
|
|
383
|
+
# the reason given at the first VALID above.
|
|
384
|
+
[LibHegel::HEGEL_STATUS_VALID, nil, nil, nil]
|
|
351
385
|
rescue Exception => e
|
|
352
386
|
[LibHegel::HEGEL_STATUS_INTERESTING, origin_for(e), e, test_case.entries]
|
|
353
387
|
end
|
data/lib/hegel/settings.rb
CHANGED
|
@@ -41,11 +41,22 @@ module Hegel
|
|
|
41
41
|
large_initial_test_case: LibHegel::HEGEL_HC_LARGE_INITIAL_TEST_CASE
|
|
42
42
|
}.freeze
|
|
43
43
|
|
|
44
|
+
NONDETERMINISM_STRICTNESS_CODES = {
|
|
45
|
+
quiet: LibHegel::HEGEL_NONDETERMINISM_QUIET,
|
|
46
|
+
warn: LibHegel::HEGEL_NONDETERMINISM_WARN,
|
|
47
|
+
error: LibHegel::HEGEL_NONDETERMINISM_ERROR
|
|
48
|
+
}.freeze
|
|
49
|
+
|
|
50
|
+
BACKEND_CODES = {
|
|
51
|
+
default: LibHegel::HEGEL_BACKEND_DEFAULT,
|
|
52
|
+
urandom: LibHegel::HEGEL_BACKEND_URANDOM
|
|
53
|
+
}.freeze
|
|
54
|
+
|
|
44
55
|
module_function
|
|
45
56
|
|
|
46
57
|
# Applies every one of Hegel.test's settings keywords to +settings+ via
|
|
47
|
-
# +impl+.
|
|
48
|
-
#
|
|
58
|
+
# +impl+. Every keyword skips its setter when nil, except
|
|
59
|
+
# database/database_key and report_multiple_failures (below).
|
|
49
60
|
# +database+/+database_key+ follow #apply_database's own table, called
|
|
50
61
|
# unconditionally since even the nil/nil case has a setter to call (see
|
|
51
62
|
# its comment). +report_multiple_failures+ is called unconditionally too,
|
|
@@ -53,26 +64,96 @@ module Hegel
|
|
|
53
64
|
# it nil, so this method never sees nil for it -- see Hegel::Runner.run's
|
|
54
65
|
# own comment for why that default departs from every other keyword's
|
|
55
66
|
# nil-means-engine-default rule.
|
|
56
|
-
def apply(impl, ctx, settings,
|
|
57
|
-
|
|
67
|
+
def apply(impl, ctx, settings, database:, database_key:, report_multiple_failures:, **keywords)
|
|
68
|
+
apply_shared(impl, ctx, settings, **keywords)
|
|
69
|
+
apply_database(impl, ctx, settings, database: database, database_key: database_key)
|
|
70
|
+
impl.settings_set_report_multiple_failures(ctx, settings, report_multiple_failures)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The keywords a run and a registered profile have in common. The example
|
|
74
|
+
# database key and report_multiple_failures are left to the callers:
|
|
75
|
+
# a run always passes the latter, and a profile snapshot carries no key.
|
|
76
|
+
def apply_shared(impl, ctx, settings, test_cases:, seed:, derandomize:, verbosity:, phases:,
|
|
77
|
+
suppress_health_check:, stop_on_first_failure:, print_blob:, show_statistics:,
|
|
78
|
+
nondeterminism_strictness:, backend:)
|
|
58
79
|
impl.settings_set_test_cases(ctx, settings, test_cases) unless test_cases.nil?
|
|
59
80
|
impl.settings_set_seed(ctx, settings, seed, true) unless seed.nil?
|
|
60
81
|
impl.settings_set_derandomize(ctx, settings, derandomize) unless derandomize.nil?
|
|
61
82
|
apply_verbosity(impl, ctx, settings, verbosity) unless verbosity.nil?
|
|
62
|
-
apply_database(impl, ctx, settings, database: database, database_key: database_key)
|
|
63
83
|
apply_phases(impl, ctx, settings, phases) unless phases.nil?
|
|
64
84
|
apply_suppress_health_check(impl, ctx, settings, suppress_health_check) unless suppress_health_check.nil?
|
|
65
|
-
|
|
85
|
+
unless stop_on_first_failure.nil?
|
|
86
|
+
impl.settings_set_stop_on_first_failure(ctx, settings, stop_on_first_failure)
|
|
87
|
+
end
|
|
88
|
+
impl.settings_set_print_blob(ctx, settings, print_blob) unless print_blob.nil?
|
|
89
|
+
impl.settings_set_show_statistics(ctx, settings, show_statistics) unless show_statistics.nil?
|
|
90
|
+
unless nondeterminism_strictness.nil?
|
|
91
|
+
code = lookup(NONDETERMINISM_STRICTNESS_CODES, nondeterminism_strictness, "nondeterminism_strictness")
|
|
92
|
+
impl.settings_set_nondeterminism_strictness(ctx, settings, code)
|
|
93
|
+
end
|
|
94
|
+
impl.settings_set_backend(ctx, settings, lookup(BACKEND_CODES, backend, "backend")) unless backend.nil?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# A settings handle for a run or a profile: from the default profile when
|
|
98
|
+
# +profile+ is nil, from the named one otherwise.
|
|
99
|
+
def new_handle(impl, ctx, profile)
|
|
100
|
+
profile.nil? ? impl.settings_new(ctx) : impl.settings_new_for_profile(ctx, profile)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Registers the settings built from +profile+ (nil: the default profile)
|
|
104
|
+
# and the keywords as the profile +name+, for the whole process. A later
|
|
105
|
+
# Hegel.test(profile: name) or set_default_profile(name) starts from them.
|
|
106
|
+
# Registration is not retroactive and replaces an earlier one of the same
|
|
107
|
+
# name. The keywords are Hegel.test's, each nil for "leave the starting
|
|
108
|
+
# value", minus database_key, which the engine leaves out of a snapshot,
|
|
109
|
+
# and with +database+ passed straight to the engine ("" disables it) and
|
|
110
|
+
# report_multiple_failures applied only when given: Hegel.test always
|
|
111
|
+
# passes its own, so a profile's value for that one never reaches a run
|
|
112
|
+
# (docs/adr/0028).
|
|
113
|
+
#
|
|
114
|
+
# The profile table is process-wide state in the engine, not on +impl+'s
|
|
115
|
+
# context, so a registration made here outlives the context this opens.
|
|
116
|
+
def register_profile(name, profile: nil, test_cases: nil, seed: nil, derandomize: nil, verbosity: nil,
|
|
117
|
+
database: nil, phases: nil, suppress_health_check: nil, report_multiple_failures: nil,
|
|
118
|
+
stop_on_first_failure: nil, print_blob: nil, show_statistics: nil, nondeterminism_strictness: nil,
|
|
119
|
+
backend: nil, impl: Hegel.default_impl)
|
|
120
|
+
LibHegel.with_context(impl) do |ctx|
|
|
121
|
+
settings = new_handle(impl, ctx, profile)
|
|
122
|
+
begin
|
|
123
|
+
apply_shared(impl, ctx, settings, test_cases: test_cases, seed: seed, derandomize: derandomize,
|
|
124
|
+
verbosity: verbosity, phases: phases, suppress_health_check: suppress_health_check,
|
|
125
|
+
stop_on_first_failure: stop_on_first_failure, print_blob: print_blob, show_statistics: show_statistics,
|
|
126
|
+
nondeterminism_strictness: nondeterminism_strictness, backend: backend)
|
|
127
|
+
impl.settings_set_database(ctx, settings, database) unless database.nil?
|
|
128
|
+
unless report_multiple_failures.nil?
|
|
129
|
+
impl.settings_set_report_multiple_failures(ctx, settings, report_multiple_failures)
|
|
130
|
+
end
|
|
131
|
+
impl.settings_register_profile(ctx, name, settings)
|
|
132
|
+
ensure
|
|
133
|
+
impl.settings_free(ctx, settings)
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Makes +name+ the profile a nil +profile+ resolves to, for the whole
|
|
140
|
+
# process, ahead of HEGEL_DEFAULT_PROFILE and hegel.toml. "" clears it.
|
|
141
|
+
# The name need not be registered yet.
|
|
142
|
+
def set_default_profile(name, impl: Hegel.default_impl)
|
|
143
|
+
LibHegel.with_context(impl) { |ctx| impl.set_default_profile(ctx, name) }
|
|
144
|
+
nil
|
|
66
145
|
end
|
|
67
146
|
|
|
68
147
|
# Split from #apply so the VERBOSITY_CODES lookup (one of several
|
|
69
148
|
# keywords that can fail) is not buried inside the top-level sequence.
|
|
70
149
|
def apply_verbosity(impl, ctx, settings, verbosity)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
150
|
+
impl.settings_set_verbosity(ctx, settings, lookup(VERBOSITY_CODES, verbosity, "verbosity"))
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def lookup(codes, value, keyword)
|
|
154
|
+
codes.fetch(value) do
|
|
155
|
+
raise Hegel::Error, "hegel: unknown #{keyword} #{value.inspect}; expected one of #{codes.keys.inspect}"
|
|
74
156
|
end
|
|
75
|
-
impl.settings_set_verbosity(ctx, settings, code)
|
|
76
157
|
end
|
|
77
158
|
|
|
78
159
|
# docs/adr/0009-turn-the-example-database-on-with-a-key.md decides this
|
|
@@ -148,12 +229,7 @@ module Hegel
|
|
|
148
229
|
"hegel: #{keyword} expects one or more of #{codes.keys.inspect}, got an empty Array"
|
|
149
230
|
end
|
|
150
231
|
|
|
151
|
-
values.reduce(0)
|
|
152
|
-
code = codes.fetch(value) do
|
|
153
|
-
raise Hegel::Error, "hegel: unknown #{keyword} #{value.inspect}; expected one of #{codes.keys.inspect}"
|
|
154
|
-
end
|
|
155
|
-
mask | code
|
|
156
|
-
end
|
|
232
|
+
values.reduce(0) { |mask, value| mask | lookup(codes, value, keyword) }
|
|
157
233
|
end
|
|
158
234
|
end
|
|
159
235
|
end
|
data/lib/hegel/state_machine.rb
CHANGED
|
@@ -23,6 +23,9 @@ module Hegel
|
|
|
23
23
|
# class's.
|
|
24
24
|
include Syntax::Methods
|
|
25
25
|
|
|
26
|
+
# One declared rule: its block, and its selection weight.
|
|
27
|
+
Rule = Data.define(:block, :weight)
|
|
28
|
+
|
|
26
29
|
# One declared invariant: its block, and whether it runs after every
|
|
27
30
|
# round rather than when the engine samples it.
|
|
28
31
|
Invariant = Data.define(:block, :always_run)
|
|
@@ -33,8 +36,17 @@ module Hegel
|
|
|
33
36
|
# instance being tested, and is handed the running Hegel::TestCase as
|
|
34
37
|
# its one argument -- ignored if the block declares no parameter, the
|
|
35
38
|
# ordinary Ruby block rule.
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
#
|
|
40
|
+
# +weight+ hints how often the engine picks this rule against the
|
|
41
|
+
# others: a positive, finite number, 1.0 by default. It is a guide, not
|
|
42
|
+
# a distribution: the engine enables a random subset of the rules in
|
|
43
|
+
# each test case, so a rule's frequency also depends on which others
|
|
44
|
+
# are enabled. The engine rejects a weight that is not finite and
|
|
45
|
+
# positive when the machine runs.
|
|
46
|
+
def rule(name, weight: 1.0, &block)
|
|
47
|
+
raise Hegel::Error, "hegel: rule weight must be a real number, got #{weight.inspect}" unless weight.is_a?(Numeric) && weight.real?
|
|
48
|
+
|
|
49
|
+
declare(:@rules, "rule", name, Rule.new(block: block, weight: weight))
|
|
38
50
|
end
|
|
39
51
|
|
|
40
52
|
# Declares an invariant named +name+, checked before the first rule
|
|
@@ -47,9 +59,10 @@ module Hegel
|
|
|
47
59
|
declare(:@invariants, "invariant", name, Invariant.new(block: block, always_run: always_run))
|
|
48
60
|
end
|
|
49
61
|
|
|
50
|
-
# name =>
|
|
62
|
+
# name => Rule, in declaration order, this class's own declarations
|
|
51
63
|
# merged over its ancestors'. Hegel::Stateful.run reads this directly
|
|
52
|
-
# to build the ordered rule-name
|
|
64
|
+
# to build the ordered rule-name and weight lists libhegel indexes by
|
|
65
|
+
# position.
|
|
53
66
|
def rule_definitions
|
|
54
67
|
merged_definitions(:@rules, :rule_definitions)
|
|
55
68
|
end
|