hecks 3.6.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hecks/bluebook/meta_validator/modes.rb +14 -0
- data/lib/hecks/bluebook/meta_validator/syntax_boot/disk_cache.rb +28 -3
- data/lib/hecks/bluebook/meta_validator/verdict_cache/prebuilt.rb +44 -0
- data/lib/hecks/bluebook/meta_validator/verdict_cache.rb +3 -17
- data/lib/hecks/bluebook/meta_validator.rb +4 -2
- data/lib/hecks/cache_dir.rb +10 -0
- data/lib/hecks/corpus/routes.rb +1 -1
- data/lib/hecks/deploy/bluebook/deploy.bluebook +109 -0
- data/lib/hecks/deploy/oidc.json +10 -0
- data/lib/hecks/doors/usage_cache.rb +19 -9
- data/lib/hecks/fuzzing/coverage_campaign.rb +42 -17
- data/lib/hecks/fuzzing/failure_triage.rb +80 -0
- data/lib/hecks/fuzzing/form_census/chapter_forms.rb +38 -0
- data/lib/hecks/fuzzing/form_census/reference_forms.rb +49 -0
- data/lib/hecks/fuzzing/form_census.rb +28 -52
- data/lib/hecks/fuzzing/isolated_boot/postgres_scratch.rb +22 -7
- data/lib/hecks/fuzzing/isolated_boot/scratch_schema.rb +27 -0
- data/lib/hecks/fuzzing/isolated_boot.rb +6 -2
- data/lib/hecks/fuzzing/mutation/checks.rb +76 -0
- data/lib/hecks/fuzzing/mutation/finders.rb +136 -0
- data/lib/hecks/fuzzing/mutation/judge.rb +94 -0
- data/lib/hecks/fuzzing/mutation/observation.rb +38 -0
- data/lib/hecks/fuzzing/mutation/operators.rb +52 -0
- data/lib/hecks/fuzzing/mutation/report.rb +78 -0
- data/lib/hecks/fuzzing/mutation/run.rb +99 -0
- data/lib/hecks/fuzzing/mutation/selection.rb +32 -0
- data/lib/hecks/fuzzing/mutation/site.rb +21 -0
- data/lib/hecks/fuzzing/mutation/source_map.rb +59 -0
- data/lib/hecks/fuzzing/mutation.rb +12 -0
- data/lib/hecks/fuzzing/properties/catalog.rb +10 -59
- data/lib/hecks/fuzzing/properties/guaranteed_by_construction.rb +63 -0
- data/lib/hecks/fuzzing/properties/read_model_names.rb +50 -0
- data/lib/hecks/fuzzing/properties/read_models.rb +138 -0
- data/lib/hecks/fuzzing/properties/role_checks.rb +50 -0
- data/lib/hecks/fuzzing/properties.rb +7 -0
- data/lib/hecks/fuzzing/replay/observation.rb +1 -1
- data/lib/hecks/fuzzing/replay/role_check.rb +49 -0
- data/lib/hecks/fuzzing/replay/session.rb +22 -6
- data/lib/hecks/fuzzing/replay.rb +1 -0
- data/lib/hecks/fuzzing/runtime_coverage.rb +70 -0
- data/lib/hecks/fuzzing/shrinker/value_simplification.rb +64 -0
- data/lib/hecks/fuzzing/shrinker.rb +3 -0
- data/lib/hecks/fuzzing.rb +3 -0
- data/lib/hecks/gate/stages.yml +12 -2
- data/lib/hecks/hecks/adapters/deploy_toolchain/answers.rb +2 -1
- data/lib/hecks/hecks/adapters/github_rulesets.rb +7 -5
- data/lib/hecks/hecks/adapters/process_pool.rb +26 -4
- data/lib/hecks/hecks/adapters/site_toolchain/prober.rb +64 -0
- data/lib/hecks/hecks/adapters/site_toolchain.rb +41 -5
- data/lib/hecks/hecks/custodian.behaviors +6 -0
- data/lib/hecks/hecks/custodian.bluebook +37 -0
- data/lib/hecks/hecks/hecks.hecksagon +6 -0
- data/lib/hecks/language/bluebook/vocabulary.bluebook +4 -1
- data/lib/hecks/projections/deploy/box/roll_scripts.rb +6 -4
- data/lib/hecks/projections/deploy/box/settings/patterns.rb +4 -0
- data/lib/hecks/projections/deploy/box/settings/types.rb +7 -3
- data/lib/hecks/projections/deploy/box/settings.rb +18 -0
- data/lib/hecks/projections/deploy/box/shared_database.rb +69 -0
- data/lib/hecks/projections/deploy/box/templates/Makefile.tmpl +6 -7
- data/lib/hecks/projections/deploy/box/templates/deploy-box.sh.tmpl +1 -1
- data/lib/hecks/projections/deploy/box/templates/provision-database.sh.tmpl +97 -0
- data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl +4 -2
- data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl +2 -1
- data/lib/hecks/projections/deploy/box.rb +14 -2
- data/lib/hecks/projections/deploy/shared_database_instance/templates/Makefile.tmpl +11 -0
- data/lib/hecks/projections/deploy/shared_database_instance/templates/README.md.tmpl +25 -0
- data/lib/hecks/projections/deploy/shared_database_instance/templates/rds.yaml.tmpl +180 -0
- data/lib/hecks/projections/deploy/shared_database_instance.rb +106 -0
- data/lib/hecks/projections/deploy/vercel/settings.rb +83 -0
- data/lib/hecks/projections/deploy/vercel.rb +134 -0
- data/lib/hecks/projections/site/probe.rb +131 -0
- data/lib/hecks/projections/site/site_host/dockerfile.rb +1 -1
- data/lib/hecks/projections/site/site_host/templates.rb +12 -4
- data/lib/hecks/projections.rb +2 -0
- data/lib/hecks/quality_control/adapters/github_checks.rb +9 -2
- data/lib/hecks/quality_control/cli/qa_sweep/dials.rb +1 -0
- data/lib/hecks/quality_control/cli/qa_sweep/seed_loop.rb +10 -2
- data/lib/hecks/runtime/value/validation.rb +34 -3
- data/lib/hecks/rust_build/native_build.rb +29 -1
- data/lib/hecks/rust_build/wasm.rb +16 -3
- data/lib/hecks/site/bluebook/site.bluebook +31 -0
- data/lib/hecks/site/bluebook/site.ports.hecksagon +6 -0
- data/lib/hecks/tools/deploy_recipe/generation.rb +59 -0
- data/lib/hecks/tools/deploy_recipe/targets.rb +56 -2
- data/lib/hecks/tools/deploy_recipe/worlds.rb +2 -1
- data/lib/hecks/tools/deploy_recipe.rb +20 -10
- data/lib/hecks/tools/fuzz_sweep/options.rb +21 -10
- data/lib/hecks/tools/fuzz_sweep/reporting.rb +25 -3
- data/lib/hecks/tools/fuzz_sweep.rb +26 -9
- data/lib/hecks/tools/lanes/files.rb +2 -4
- data/lib/hecks/tools/lanes/live.rb +3 -4
- data/lib/hecks/tools/lanes/ruleset.rb +23 -1
- data/lib/hecks/tools/lanes.rb +12 -1
- data/lib/hecks/tools/mutation_run.rb +73 -0
- data/lib/hecks/tools.rb +1 -0
- data/lib/hecks/version.rb +1 -1
- data/lib/hecks/vocabulary.rb +3 -1
- data/prebuilt/4e6c3cd1f578a38732ea5454dae52562d03d8e9886216077a5c34cbf589f3463.marshal +0 -0
- data/prebuilt/911cdcc56b5a094ed26583a4c76401dba6455b1f434722ed22b850bc1b1bdd28.marshal +0 -0
- data/prebuilt/verdicts-3230a251570aa03666708d16f92e646feab6a3f8cc2a3540c017d748e41d41a1.json +1 -0
- data/qa/settings.yml +1 -0
- data/rust/codegen/src/fielded.rs +3 -3
- data/rust/codegen/src/json_codec.rs +2 -2
- data/rust/codegen/src/naming.rs +20 -0
- data/rust/host/Cargo.toml +6 -2
- data/rust/host/HECKS_RELEASE +1 -1
- data/rust/host/src/api.rs +28 -5
- data/rust/host/src/auth.rs +11 -1
- data/rust/host/src/boundary_fuzz/api.rs +260 -0
- data/rust/host/src/boundary_fuzz/auth.rs +119 -0
- data/rust/host/src/boundary_fuzz/expr_json.rs +158 -0
- data/rust/host/src/boundary_fuzz/ir.rs +79 -0
- data/rust/host/src/boundary_fuzz/server.rs +227 -0
- data/rust/host/src/boundary_fuzz/web.rs +125 -0
- data/rust/host/src/commerce_ir.rs +459 -0
- data/rust/host/src/extension.rs +100 -0
- data/rust/host/src/fuzz_support.rs +165 -0
- data/rust/host/src/ir.rs +4 -448
- data/rust/host/src/journal.rs +1 -1
- data/rust/host/src/lib.rs +39 -0
- data/rust/host/src/main.rs +10 -44
- data/rust/host/src/mint.rs +1 -1
- data/rust/host/src/payments/tests.rs +8 -8
- data/rust/host/src/payments.rs +10 -10
- data/rust/host/src/rate_limit.rs +27 -41
- data/rust/host/src/secrets.rs +44 -0
- data/rust/host/src/server.rs +6 -1
- data/rust/host/src/web/commerce.rs +113 -0
- data/rust/host/src/web/newsletter.rs +2 -1
- data/rust/host/src/web/newsletter_send.rs +1 -1
- data/rust/host/src/web/registration_receipt.rs +2 -2
- data/rust/host/src/web/registrations.rs +49 -49
- data/rust/host/src/web/resend_webhook.rs +299 -0
- data/rust/host/src/web.rs +55 -49
- data/rust/parser/fuzz/.gitignore +4 -0
- data/rust/parser/fuzz/Cargo.lock +89 -0
- data/rust/parser/fuzz/Cargo.toml +27 -0
- data/rust/parser/fuzz/README.md +19 -0
- data/rust/parser/fuzz/fuzz_targets/chapter.rs +50 -0
- data/rust/parser/fuzz/fuzz_targets/resolve.rs +35 -0
- data/rust/parser/src/parse/command.rs +14 -2
- data/rust/parser/src/parse/mod.rs +25 -0
- data/rust/parser/src/ruby_value.rs +95 -6
- data/rust/src/kernel/json.rs +34 -3
- metadata +58 -2
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
require_relative "site"
|
|
2
|
+
require_relative "source_map"
|
|
3
|
+
require_relative "finders"
|
|
4
|
+
|
|
5
|
+
module Hecks
|
|
6
|
+
module Fuzzing
|
|
7
|
+
module Mutation
|
|
8
|
+
# Finds every site a mutation operator applies to in a bluebook's source.
|
|
9
|
+
#
|
|
10
|
+
# Each operator is a small change a reviewer would call a plausible slip: a rule dropped,
|
|
11
|
+
# a comparison off by one, a guard loosened, a handler forgotten. A site is a candidate
|
|
12
|
+
# only; the mutant it makes may not boot, and `Run` discards one that does not.
|
|
13
|
+
module Operators
|
|
14
|
+
# What each operator does, in the words a report uses.
|
|
15
|
+
CATALOG = {
|
|
16
|
+
drop_given: "removes a `given` guard",
|
|
17
|
+
drop_invariant: "removes an `invariant`",
|
|
18
|
+
flip_comparison: "moves a comparison one step (`>=` to `>`, `<` to `<=`, `==` to `!=`)",
|
|
19
|
+
flip_query_bound: "swaps a query's `lt` and `gt` bound",
|
|
20
|
+
drop_pattern: "removes an attribute's `pattern:`",
|
|
21
|
+
drop_one_of: "removes an attribute's `one_of:` closed set",
|
|
22
|
+
drop_sets: "removes a `sets` line, so the command no longer records that field",
|
|
23
|
+
drop_from_guard: "removes a `from:` lifecycle guard",
|
|
24
|
+
retarget_transition: "points a lifecycle transition at a different state",
|
|
25
|
+
drop_policy: "removes a whole `policy`",
|
|
26
|
+
drop_saga_transition: "removes a `process_manager` transition and its handler",
|
|
27
|
+
drop_saga_dispatch: "removes a `dispatch` a `process_manager` handler makes",
|
|
28
|
+
swap_emits: "makes a command emit a different event of the same file"
|
|
29
|
+
}.freeze
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# @param files [Hash{String => String}] bluebook source by path, relative to the domain
|
|
34
|
+
# directory
|
|
35
|
+
# @return [Array<Site>] every site of every operator, in file and line order
|
|
36
|
+
def sites(files)
|
|
37
|
+
files.sort.flat_map { |file, source| sites_in(SourceMap.new(file, source.lines)) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @param map [SourceMap] one bluebook
|
|
41
|
+
# @return [Array<Site>] its sites
|
|
42
|
+
def sites_in(map)
|
|
43
|
+
map.lines.each_index.flat_map do |index|
|
|
44
|
+
Finders::ALL.flat_map { |finder| finder.at(map, index) }.map do |change|
|
|
45
|
+
Site.new(file: map.file, line: index + 1, original: map.lines[index], **change)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Mutation
|
|
4
|
+
# What a mutation run tried and how each mutant ended.
|
|
5
|
+
#
|
|
6
|
+
# The score is killed mutants over killed plus survived: the share of behavior-changing
|
|
7
|
+
# mutants the checks caught. Unreached and invalid mutants say nothing about the checks, so
|
|
8
|
+
# they stay out of it.
|
|
9
|
+
class Report
|
|
10
|
+
# What the run was, for the report's heading.
|
|
11
|
+
#
|
|
12
|
+
# @!attribute [r] domain
|
|
13
|
+
# @return [String] the domain's name
|
|
14
|
+
# @!attribute [r] seed
|
|
15
|
+
# @return [Integer] the run's seed
|
|
16
|
+
# @!attribute [r] available
|
|
17
|
+
# @return [Integer] how many sites the operators found in all
|
|
18
|
+
# @!attribute [r] sequences
|
|
19
|
+
# @return [Integer] how many generated sequences each mutant was replayed against
|
|
20
|
+
# @!attribute [r] corpus
|
|
21
|
+
# @return [Boolean] whether a corpus script was replayed too
|
|
22
|
+
Facts = Struct.new(:domain, :seed, :available, :sequences, :corpus)
|
|
23
|
+
|
|
24
|
+
SURVIVED = "SURVIVED — behavior changed and no property, corpus expectation or behaviors test failed:".freeze
|
|
25
|
+
UNREACHED = "UNREACHED — no sequence told these apart from the original (equivalent, or never exercised):".freeze
|
|
26
|
+
|
|
27
|
+
attr_reader :facts, :verdicts
|
|
28
|
+
|
|
29
|
+
# @param facts [Facts] what the run was
|
|
30
|
+
# @param verdicts [Array<Outcome>] one per mutant tried
|
|
31
|
+
def initialize(facts, verdicts)
|
|
32
|
+
@facts = facts
|
|
33
|
+
@verdicts = verdicts
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
%i[killed survived unreached invalid].each do |status|
|
|
37
|
+
define_method(status) { verdicts.select { |outcome| outcome.status == status } }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @return [Float, nil] killed / (killed + survived), nil when no mutant changed behavior
|
|
41
|
+
def score
|
|
42
|
+
judged = killed.size + survived.size
|
|
43
|
+
judged.zero? ? nil : killed.size.fdiv(judged)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param minimum [Float] the lowest score that still passes
|
|
47
|
+
# @return [Boolean] whether the score is at least `minimum`; a run with nothing judged
|
|
48
|
+
# passes
|
|
49
|
+
def passes?(minimum) = score.nil? || score >= minimum
|
|
50
|
+
|
|
51
|
+
# @return [String] the report as the command prints it
|
|
52
|
+
def to_s
|
|
53
|
+
[title, tally, *section(SURVIVED, survived), *section(UNREACHED, unreached)].join("\n")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def title
|
|
59
|
+
score_text = score ? format("%.0f%%", score * 100) : "n/a"
|
|
60
|
+
"mutation #{facts.domain} (seed #{facts.seed}): #{verdicts.size} of #{facts.available} mutants tried " \
|
|
61
|
+
"against #{facts.sequences} sequence(s)#{" and the corpus script" if facts.corpus} — score #{score_text}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def tally
|
|
65
|
+
" killed #{killed.size}, survived #{survived.size}, unreached #{unreached.size}, invalid #{invalid.size}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def section(heading, outcomes)
|
|
69
|
+
outcomes.empty? ? [] : ["", heading, *outcomes.map { |outcome| describe(outcome.site) }]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def describe(site)
|
|
73
|
+
" #{site.id} #{Operators::CATALOG.fetch(site.operator)}\n was: #{site.original.strip}"
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
require "json"
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require_relative "operators"
|
|
5
|
+
require_relative "judge"
|
|
6
|
+
require_relative "selection"
|
|
7
|
+
require_relative "report"
|
|
8
|
+
|
|
9
|
+
module Hecks
|
|
10
|
+
module Fuzzing
|
|
11
|
+
module Mutation
|
|
12
|
+
# Mutation testing of a domain's own checks: makes each small change `Operators` finds to a
|
|
13
|
+
# copy of the domain, replays the same generated sequences, the domain's corpus script (when
|
|
14
|
+
# it has one) and its `.behaviors` tests against the copy, and asks whether any check noticed.
|
|
15
|
+
#
|
|
16
|
+
# A mutant is killed when a fuzz property fails, the replay crashes, a corpus expectation
|
|
17
|
+
# goes unmet or a behaviors test fails. It survives when its behavior differs from the
|
|
18
|
+
# unmutated domain's and nothing failed: the checks have a hole there. It is unreached when
|
|
19
|
+
# no sequence told it apart from the original, which is either an equivalent change or a
|
|
20
|
+
# part of the domain the generator never gets to. A mutant that does not boot is invalid and
|
|
21
|
+
# counts toward no score.
|
|
22
|
+
#
|
|
23
|
+
# Deterministic: the sequences and the choice of mutants both come from `seed`.
|
|
24
|
+
class Run
|
|
25
|
+
# The checkout this file lives in, where a domain's corpus script is looked for.
|
|
26
|
+
ROOT = File.expand_path("../../../..", __dir__)
|
|
27
|
+
|
|
28
|
+
# The knobs of a run: `seed` draws the sequences and the choice of mutants; `budget` caps
|
|
29
|
+
# the mutants tried; `seeds` and `steps` size the generated sequences each mutant is
|
|
30
|
+
# replayed against; `corpus` is a script to replay too (found by the domain's name when
|
|
31
|
+
# nil); `operators` restricts the run to those operators when given.
|
|
32
|
+
Settings = Struct.new(:seed, :budget, :seeds, :steps, :corpus, :operators, keyword_init: true)
|
|
33
|
+
|
|
34
|
+
DEFAULTS = { seed: 1, budget: 25, seeds: 3, steps: 30, corpus: nil, operators: nil }.freeze
|
|
35
|
+
|
|
36
|
+
# @param domain [String] the domain directory
|
|
37
|
+
# @param options [Hash] any of `Settings`' members
|
|
38
|
+
# @return [Report] what was tried and how each mutant ended
|
|
39
|
+
def self.call(domain, **) = new(domain, Settings.new(**DEFAULTS, **)).call
|
|
40
|
+
|
|
41
|
+
def initialize(domain, settings)
|
|
42
|
+
@domain = domain.chomp("/")
|
|
43
|
+
@settings = settings
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Report] the mutants tried, each with its verdict
|
|
47
|
+
def call
|
|
48
|
+
plan = Plan.new(generate_sequences, corpus_script)
|
|
49
|
+
baseline = Checks.baseline(@domain, plan)
|
|
50
|
+
sites = Operators.sites(bluebook_sources)
|
|
51
|
+
verdicts = chosen(sites).map { |site| judge(site, plan, baseline) }
|
|
52
|
+
Report.new(facts(sites.size, plan), verdicts)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def facts(available, plan)
|
|
58
|
+
Report::Facts.new(File.basename(@domain), @settings.seed, available, plan.sequences.size, !plan.corpus.nil?)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def chosen(sites)
|
|
62
|
+
Selection.pick(sites, budget: @settings.budget, seed: @settings.seed, operators: @settings.operators)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The same sequences for every mutant, generated against the unmutated domain so each is
|
|
66
|
+
# valid there and any difference is the mutation's.
|
|
67
|
+
def generate_sequences
|
|
68
|
+
Array.new(@settings.seeds) do |index|
|
|
69
|
+
SequenceGenerator.generate(@domain, seed: @settings.seed + index, steps: @settings.steps,
|
|
70
|
+
adversarial: index.odd? ? 0.3 : 0.0)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def judge(site, plan, baseline)
|
|
75
|
+
Dir.mktmpdir("hecks-mutant") do |tmp|
|
|
76
|
+
copy = File.join(tmp, File.basename(@domain))
|
|
77
|
+
FileUtils.cp_r(@domain, copy)
|
|
78
|
+
path = File.join(copy, site.file)
|
|
79
|
+
File.write(path, site.apply(File.readlines(path)).join)
|
|
80
|
+
Judge.call(site, copy, plan, baseline)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @return [Hash{String => String}] each bluebook of the domain by path relative to it
|
|
85
|
+
def bluebook_sources
|
|
86
|
+
Dir.glob(File.join(@domain, "**", "*.bluebook")).to_h do |path|
|
|
87
|
+
[path.delete_prefix("#{@domain}/"), File.read(path)]
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The parsed corpus script: the one asked for, else the one of this domain's name.
|
|
92
|
+
def corpus_script
|
|
93
|
+
path = @settings.corpus || File.join(ROOT, "spec", "corpus", "#{File.basename(@domain)}.json")
|
|
94
|
+
File.file?(path) ? JSON.parse(File.read(path)) : nil
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Mutation
|
|
4
|
+
# Chooses which sites to try within a budget: every operator gets a turn before any gets a
|
|
5
|
+
# second, each in a seeded shuffle, so a small budget still samples all of them instead of
|
|
6
|
+
# exhausting the first.
|
|
7
|
+
module Selection
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
# @param sites [Array<Site>] every site found
|
|
11
|
+
# @param budget [Integer] how many to take at most
|
|
12
|
+
# @param seed [Integer] draws the shuffle
|
|
13
|
+
# @param operators [Array<Symbol>, nil] restricts the choice to these operators
|
|
14
|
+
# @return [Array<Site>] the sites to try, repeatable for one seed
|
|
15
|
+
def pick(sites, budget:, seed:, operators: nil)
|
|
16
|
+
sites = sites.select { |site| operators.include?(site.operator) } if operators
|
|
17
|
+
rng = Random.new(seed)
|
|
18
|
+
queues = sites.group_by(&:operator).sort_by { |operator, _| operator.to_s }
|
|
19
|
+
.map { |_, group| group.shuffle(random: rng) }
|
|
20
|
+
deal(queues, budget)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Takes one from each queue in turn until the budget is spent or they are empty.
|
|
24
|
+
def deal(queues, budget)
|
|
25
|
+
taken = []
|
|
26
|
+
taken.concat(queues.filter_map(&:shift)) while taken.size < budget && queues.any?(&:any?)
|
|
27
|
+
taken.first(budget)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Mutation
|
|
4
|
+
# One place a small semantic change can be made to a bluebook's source: which `operator`
|
|
5
|
+
# (a key of `Operators::CATALOG`), in which `file` (relative to the domain directory), at
|
|
6
|
+
# which `line` (the first it touches, from 1), how many lines it `removed` from there, the
|
|
7
|
+
# `replacement` lines that stand in their place (none for a pure removal) and the `original`
|
|
8
|
+
# line, for a report.
|
|
9
|
+
Site = Struct.new(:operator, :file, :line, :removed, :replacement, :original, keyword_init: true) do
|
|
10
|
+
# @return [String] the stable name of this mutant, `operator@file:line`
|
|
11
|
+
def id = "#{operator}@#{file}:#{line}"
|
|
12
|
+
|
|
13
|
+
# @param lines [Array<String>] the bluebook's lines, newline-terminated
|
|
14
|
+
# @return [Array<String>] the lines with this change made
|
|
15
|
+
def apply(lines)
|
|
16
|
+
lines.dup.tap { |changed| changed[line - 1, removed] = replacement }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Mutation
|
|
4
|
+
# What a bluebook's source says about itself, so a finder can ask where a line sits: the block
|
|
5
|
+
# keyword it is inside, how long the block that starts on it runs, and which lifecycle states
|
|
6
|
+
# and events the file names.
|
|
7
|
+
class SourceMap
|
|
8
|
+
# A line that opens a block, as `policy "X" do` or `command "Y", from: "z" do |a|`.
|
|
9
|
+
OPENER = /\A(\s*)(\w+)\b.*\bdo\s*(\|[^|]*\|)?\s*\z/
|
|
10
|
+
TRANSITION = /\A(\s*)transition\s+"(\w+)"\s*=>\s*"([^"]+)"(.*?)\s*\z/
|
|
11
|
+
EMITS = /\A(\s*)emits\s+(\w+)\s*\z/
|
|
12
|
+
|
|
13
|
+
attr_reader :file, :lines
|
|
14
|
+
|
|
15
|
+
# @param file [String] the bluebook's path relative to the domain directory
|
|
16
|
+
# @param lines [Array<String>] its lines, newline-terminated
|
|
17
|
+
def initialize(file, lines)
|
|
18
|
+
@file = file
|
|
19
|
+
@lines = lines
|
|
20
|
+
@scopes = compute_scopes
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @param index [Integer] a line's position, from 0
|
|
24
|
+
# @return [String, nil] the keyword of the innermost block the line is inside
|
|
25
|
+
def scope(index) = @scopes[index]
|
|
26
|
+
|
|
27
|
+
# @return [Array<String>] every state a `transition` in the file moves to
|
|
28
|
+
def states
|
|
29
|
+
@states ||= lines.filter_map { |line| line[TRANSITION, 3] }.uniq
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [Array<String>] every event name a command in the file emits
|
|
33
|
+
def emitted
|
|
34
|
+
@emitted ||= lines.filter_map { |line| line[EMITS, 2] }.uniq
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @param index [Integer] the position of the line that opens a block
|
|
38
|
+
# @return [Integer, nil] the lines from it to its `end`, nil when the block never closes
|
|
39
|
+
def block_length(index)
|
|
40
|
+
indent = lines[index][/\A\s*/]
|
|
41
|
+
close = ((index + 1)...lines.length).find { |at| lines[at].match?(/\A#{indent}end\b/) }
|
|
42
|
+
close && ((close - index) + 1)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def compute_scopes
|
|
48
|
+
stack = []
|
|
49
|
+
lines.map do |line|
|
|
50
|
+
stack.pop while stack.any? && line.match?(/\A#{stack.last[0]}end\b/)
|
|
51
|
+
inside = stack.last&.last
|
|
52
|
+
(match = line.match(OPENER)) && stack.push([match[1], match[2]])
|
|
53
|
+
inside
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
require_relative "../fuzzing"
|
|
2
|
+
require_relative "mutation/run"
|
|
3
|
+
|
|
4
|
+
module Hecks
|
|
5
|
+
module Fuzzing
|
|
6
|
+
# Mutation testing of a domain's own checks: small semantic changes to its bluebooks, each run
|
|
7
|
+
# past the fuzz properties and the corpus script to see whether anything fails. A mutant nothing
|
|
8
|
+
# fails on is a hole in the checks. See `Mutation::Run`.
|
|
9
|
+
module Mutation
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -23,7 +23,10 @@ module Hecks
|
|
|
23
23
|
:outbox_rows_match_reactions,
|
|
24
24
|
:policy_reactions_follow_declared_wiring,
|
|
25
25
|
:declared_undelivered_policies_stay_undelivered,
|
|
26
|
-
:references_resolve_to_earlier_records
|
|
26
|
+
:references_resolve_to_earlier_records,
|
|
27
|
+
:role_checks_agree_with_grants,
|
|
28
|
+
:read_model_names_resolve_uniquely,
|
|
29
|
+
:read_model_heads_compose_from_references
|
|
27
30
|
].freeze
|
|
28
31
|
|
|
29
32
|
# Which language feature each property is answerable for — exhaustive of
|
|
@@ -69,70 +72,18 @@ module Hecks
|
|
|
69
72
|
declared_undelivered_policies_stay_undelivered: %w[Policy#expect_undelivered],
|
|
70
73
|
# An accepted referencing command addressed a record an earlier event created.
|
|
71
74
|
references_resolve_to_earlier_records: %w[Command#references],
|
|
75
|
+
# The grants read back through the verb the authorization provider declares in `provides`.
|
|
76
|
+
role_checks_agree_with_grants: %w[Bluebook#provides],
|
|
77
|
+
read_model_names_resolve_uniquely: %w[ReadModel#query_name],
|
|
78
|
+
# Each head's rows recomputed from the references; options apply to targeted heads only.
|
|
79
|
+
read_model_heads_compose_from_references: %w[ReadModel#reference_name ReadModel#reference_target
|
|
80
|
+
ReadModel#aggregate_heads ReadModel#options],
|
|
72
81
|
# No feature string exists for what this reads: an argument's own
|
|
73
82
|
# `relationship` (Argument is a value object, outside the meta-domain
|
|
74
83
|
# walk). Its declaration side, `Query#options`, is already claimed by
|
|
75
84
|
# authorize_scopes_or_refuses.
|
|
76
85
|
commands_respect_tenant_scope: []
|
|
77
86
|
}.freeze
|
|
78
|
-
|
|
79
|
-
# Features a replay could never observe violated, because construction
|
|
80
|
-
# makes the violation impossible to produce — not untested, but
|
|
81
|
-
# unfalsifiable. Each entry names the runtime path that guarantees it;
|
|
82
|
-
# add one only after reading and confirming that path (mirrors the
|
|
83
|
-
# discipline spec/fuzzing/meta_domain_coverage_spec.rb's KNOWN_GAPS demands
|
|
84
|
-
# in the other direction).
|
|
85
|
-
GUARANTEED_BY_CONSTRUCTION = {
|
|
86
|
-
"Aggregate#attributes" => "every field's pattern/closed-set/type passes through Value.build's one coercion " \
|
|
87
|
-
"door (value/coercion.rb#check_patterns, value/admission.rb) before it can exist " \
|
|
88
|
-
"— a stored value that violated its own declared shape was never producible to " \
|
|
89
|
-
"begin with",
|
|
90
|
-
"Aggregate#value_objects" => "the shape being coerced above — same door, same guarantee",
|
|
91
|
-
# S17, ADR 0026 — saga_advances_follow_declared_handlers already walks
|
|
92
|
-
# this list to find event_type/from_state/to_state; same door, same guarantee.
|
|
93
|
-
"ProcessManager#handlers" => "saga_advances_follow_declared_handlers already walks this list to find " \
|
|
94
|
-
"event_type/from_state/to_state — same door, same guarantee",
|
|
95
|
-
"Aggregate#identified_by" => "CommandInterpreter's data-driven dispatch order refuses AlreadyExists " \
|
|
96
|
-
"(command_interpreter.rb, command.creates?) for every creating command uniformly, " \
|
|
97
|
-
"before a duplicate id can ever be stored — collision is refused at the door, not " \
|
|
98
|
-
"produced and later caught",
|
|
99
|
-
"Entity#identified_by" => "EntityElement.check_entity_collision (runtime/entity_element.rb, moved there " \
|
|
100
|
-
"BUG#145 so both call sites share it) checks Array(current) against every part " \
|
|
101
|
-
"of the entity's own identity before an append can land — MutationApplier#" \
|
|
102
|
-
"entity_element's aggregate-owned call (Workspace.boards, on both branches " \
|
|
103
|
-
"identity arrives by: caller-supplied, or composite) AND EntityElement#" \
|
|
104
|
-
"appended_to_element's entity-owned, nested-one-hop-further call (Board.cards — " \
|
|
105
|
-
"unconditional, no auto-mint branch exists at that depth) — the same " \
|
|
106
|
-
"AlreadyExists refusal Aggregate#identified_by gets above, one or two levels " \
|
|
107
|
-
"down. Auto-minted (aggregate-owned) entities never reach the check " \
|
|
108
|
-
"(current.size + 1 can't repeat unless something remove:s from the list between " \
|
|
109
|
-
"mints, which no real domain does today — see the comment on #entity_element " \
|
|
110
|
-
"itself)",
|
|
111
|
-
"Command#attributes" => "command arguments are coerced through the SAME Value.build door as any other " \
|
|
112
|
-
"attribute — an accepted dispatch's own args already passed pattern/admits/invariant checks",
|
|
113
|
-
"Command#emits" => "CommandRules::Emission#emit iterates command.emits ITSELF to construct every " \
|
|
114
|
-
"announced Event (command_rules/emission.rb) — there is no other path to emit, so " \
|
|
115
|
-
"a command can never announce a name its own declaration doesn't list",
|
|
116
|
-
"Query#attributes" => "query arguments are coerced through the same Value.build door — same guarantee as " \
|
|
117
|
-
"Command#attributes",
|
|
118
|
-
"Query#returns" => "every row a port answers is built as the returned value object by Value.build " \
|
|
119
|
-
"(QueryInterpreter#shaped) before it enters the domain — an answer of any other " \
|
|
120
|
-
"shape is refused, so none can be accepted",
|
|
121
|
-
"Entity#attributes" => "same coercion door, one level in — an entity's own attributes are Value-typed exactly " \
|
|
122
|
-
"the way an aggregate's are",
|
|
123
|
-
"ValueObject#attributes" => "the shape Value.build enforces IS this declaration — the guarantee and the " \
|
|
124
|
-
"feature are the same fact seen from two sides",
|
|
125
|
-
"ValueObject#invariants" => "run inside the SAME coercion call (coercion.rb, before construction returns) " \
|
|
126
|
-
"that pattern-checks a VO's fields — a VO whose invariant did not hold could not " \
|
|
127
|
-
"finish being built",
|
|
128
|
-
"ValueObject#rows" => "closed-set membership is checked in value/admission.rb, the second half of the same " \
|
|
129
|
-
"one construction door",
|
|
130
|
-
# S17, ADR 0026 — Member is a genuine entity, so this reads Member#pairs,
|
|
131
|
-
# not Member#shape. ValueObject#members is the same fact ValueObject#rows
|
|
132
|
-
# counts, seen from the other side.
|
|
133
|
-
"ValueObject#members" => "the members list IS what ValueObject#rows counts — same door, same guarantee",
|
|
134
|
-
"Member#pairs" => "one level into ValueObject#rows — same door"
|
|
135
|
-
}.freeze
|
|
136
87
|
end
|
|
137
88
|
end
|
|
138
89
|
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Properties
|
|
4
|
+
# Features a replay could never observe violated, because construction
|
|
5
|
+
# makes the violation impossible to produce — not untested, but
|
|
6
|
+
# unfalsifiable. Each entry names the runtime path that guarantees it;
|
|
7
|
+
# add one only after reading and confirming that path (mirrors the
|
|
8
|
+
# discipline spec/fuzzing/meta_domain_coverage_spec.rb's KNOWN_GAPS demands
|
|
9
|
+
# in the other direction).
|
|
10
|
+
GUARANTEED_BY_CONSTRUCTION = {
|
|
11
|
+
"Aggregate#attributes" => "every field's pattern/closed-set/type passes through Value.build's one coercion " \
|
|
12
|
+
"door (value/coercion.rb#check_patterns, value/admission.rb) before it can exist " \
|
|
13
|
+
"— a stored value that violated its own declared shape was never producible to " \
|
|
14
|
+
"begin with",
|
|
15
|
+
"Aggregate#value_objects" => "the shape being coerced above — same door, same guarantee",
|
|
16
|
+
# S17, ADR 0026 — saga_advances_follow_declared_handlers already walks
|
|
17
|
+
# this list to find event_type/from_state/to_state; same door, same guarantee.
|
|
18
|
+
"ProcessManager#handlers" => "saga_advances_follow_declared_handlers already walks this list to find " \
|
|
19
|
+
"event_type/from_state/to_state — same door, same guarantee",
|
|
20
|
+
"Aggregate#identified_by" => "CommandInterpreter's data-driven dispatch order refuses AlreadyExists " \
|
|
21
|
+
"(command_interpreter.rb, command.creates?) for every creating command uniformly, " \
|
|
22
|
+
"before a duplicate id can ever be stored — collision is refused at the door, not " \
|
|
23
|
+
"produced and later caught",
|
|
24
|
+
"Entity#identified_by" => "EntityElement.check_entity_collision (runtime/entity_element.rb, moved there " \
|
|
25
|
+
"BUG#145 so both call sites share it) checks Array(current) against every part " \
|
|
26
|
+
"of the entity's own identity before an append can land — MutationApplier#" \
|
|
27
|
+
"entity_element's aggregate-owned call (Workspace.boards, on both branches " \
|
|
28
|
+
"identity arrives by: caller-supplied, or composite) AND EntityElement#" \
|
|
29
|
+
"appended_to_element's entity-owned, nested-one-hop-further call (Board.cards — " \
|
|
30
|
+
"unconditional, no auto-mint branch exists at that depth) — the same " \
|
|
31
|
+
"AlreadyExists refusal Aggregate#identified_by gets above, one or two levels " \
|
|
32
|
+
"down. Auto-minted (aggregate-owned) entities never reach the check " \
|
|
33
|
+
"(current.size + 1 can't repeat unless something remove:s from the list between " \
|
|
34
|
+
"mints, which no real domain does today — see the comment on #entity_element " \
|
|
35
|
+
"itself)",
|
|
36
|
+
"Command#attributes" => "command arguments are coerced through the SAME Value.build door as any other " \
|
|
37
|
+
"attribute — an accepted dispatch's own args already passed pattern/admits/invariant checks",
|
|
38
|
+
"Command#emits" => "CommandRules::Emission#emit iterates command.emits ITSELF to construct every " \
|
|
39
|
+
"announced Event (command_rules/emission.rb) — there is no other path to emit, so " \
|
|
40
|
+
"a command can never announce a name its own declaration doesn't list",
|
|
41
|
+
"Query#attributes" => "query arguments are coerced through the same Value.build door — same guarantee as " \
|
|
42
|
+
"Command#attributes",
|
|
43
|
+
"Query#returns" => "every row a port answers is built as the returned value object by Value.build " \
|
|
44
|
+
"(QueryInterpreter#shaped) before it enters the domain — an answer of any other " \
|
|
45
|
+
"shape is refused, so none can be accepted",
|
|
46
|
+
"Entity#attributes" => "same coercion door, one level in — an entity's own attributes are Value-typed exactly " \
|
|
47
|
+
"the way an aggregate's are",
|
|
48
|
+
"ValueObject#attributes" => "the shape Value.build enforces IS this declaration — the guarantee and the " \
|
|
49
|
+
"feature are the same fact seen from two sides",
|
|
50
|
+
"ValueObject#invariants" => "run inside the SAME coercion call (coercion.rb, before construction returns) " \
|
|
51
|
+
"that pattern-checks a VO's fields — a VO whose invariant did not hold could not " \
|
|
52
|
+
"finish being built",
|
|
53
|
+
"ValueObject#rows" => "closed-set membership is checked in value/admission.rb, the second half of the same " \
|
|
54
|
+
"one construction door",
|
|
55
|
+
# S17, ADR 0026 — Member is a genuine entity, so this reads Member#pairs,
|
|
56
|
+
# not Member#shape. ValueObject#members is the same fact ValueObject#rows
|
|
57
|
+
# counts, seen from the other side.
|
|
58
|
+
"ValueObject#members" => "the members list IS what ValueObject#rows counts — same door, same guarantee",
|
|
59
|
+
"Member#pairs" => "one level into ValueObject#rows — same door"
|
|
60
|
+
}.freeze
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Fuzzing
|
|
3
|
+
module Properties
|
|
4
|
+
# Property: how each declared read model is addressed.
|
|
5
|
+
module ReadModelNames
|
|
6
|
+
# Every read model resolves from either spelling of its name, to itself, and no two read
|
|
7
|
+
# models of one chapter share a `query_name`.
|
|
8
|
+
#
|
|
9
|
+
# @param history [Hash] a replayed history, as returned by `Fuzzing::Replay.call`
|
|
10
|
+
# @return [true, String] true, or a message naming each read model that drifts
|
|
11
|
+
def read_model_names_resolve_uniquely(history)
|
|
12
|
+
offenders = (history[:bluebooks] || {}).each_value.flat_map { |bluebook| name_offenders(bluebook) }
|
|
13
|
+
offenders.empty? || offenders.join("; ")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def name_offenders(bluebook)
|
|
17
|
+
models = bluebook.read_models
|
|
18
|
+
models.flat_map { |model| model_name_offenders(bluebook, model) } + shared_query_names(bluebook, models)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def model_name_offenders(bluebook, model)
|
|
22
|
+
[drifted_name(model), unresolved_name(bluebook, model)].compact
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def drifted_name(model)
|
|
26
|
+
return if model.query_name == snake_of(model.name)
|
|
27
|
+
|
|
28
|
+
"#{model.name}'s query_name is #{model.query_name.inspect}, not #{snake_of(model.name).inspect}"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def unresolved_name(bluebook, model)
|
|
32
|
+
return if [model.name, model.query_name].all? { |spelling| bluebook.read_model(spelling).equal?(model) }
|
|
33
|
+
|
|
34
|
+
"#{model.name} does not resolve to itself by name or by query_name"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def shared_query_names(bluebook, models)
|
|
38
|
+
models.group_by(&:query_name).select { |_name, held| held.size > 1 }.map do |name, held|
|
|
39
|
+
"#{bluebook.name} declares #{held.map(&:name).join(" and ")} under one query name #{name.inspect}"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Written out here rather than called, so a drift in `Naming.snake` is a finding.
|
|
44
|
+
def snake_of(text)
|
|
45
|
+
text.to_s.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|