saboteur 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +113 -1
- data/README.md +248 -63
- data/lib/generators/saboteur/install/install_generator.rb +21 -0
- data/lib/saboteur/agent_skill.rb +40 -0
- data/lib/saboteur/cli.rb +150 -22
- data/lib/saboteur/config.rb +27 -15
- data/lib/saboteur/config_file.rb +97 -0
- data/lib/saboteur/custom_map.rb +170 -0
- data/lib/saboteur/doctor.rb +17 -1
- data/lib/saboteur/engine/baseline_guard.rb +20 -3
- data/lib/saboteur/engine/boot.rb +9 -0
- data/lib/saboteur/engine/coverage_store.rb +31 -4
- data/lib/saboteur/engine/data_inputs.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +28 -0
- data/lib/saboteur/engine/diff_scope.rb +8 -0
- data/lib/saboteur/engine/run_fingerprint.rb +34 -6
- data/lib/saboteur/engine/scheduler.rb +7 -16
- data/lib/saboteur/engine/suite_files.rb +8 -1
- data/lib/saboteur/engine.rb +74 -13
- data/lib/saboteur/execution/reevaluator.rb +7 -4
- data/lib/saboteur/execution/supervisor.rb +23 -11
- data/lib/saboteur/frameworks/rails.rb +329 -13
- data/lib/saboteur/help.rb +33 -17
- data/lib/saboteur/ignore_list.rb +71 -17
- data/lib/saboteur/install.rb +132 -0
- data/lib/saboteur/integration/rspec.rb +47 -3
- data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
- data/lib/saboteur/mutation_source/builtin.rb +8 -2
- data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
- data/lib/saboteur/replay.rb +1 -1
- data/lib/saboteur/reporter/agent.rb +87 -0
- data/lib/saboteur/reporter/agent_clipping.rb +29 -0
- data/lib/saboteur/reporter/agent_commands.rb +61 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +37 -0
- data/lib/saboteur/reporter/agent_summary.rb +131 -0
- data/lib/saboteur/reporter/findings.rb +8 -0
- data/lib/saboteur/reporter/html.rb +49 -1
- data/lib/saboteur/reporter/stryker.rb +11 -4
- data/lib/saboteur/reporter/terminal.rb +65 -2
- data/lib/saboteur/result.rb +46 -5
- data/lib/saboteur/score.rb +89 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +48 -0
- data/lib/saboteur.rb +4 -0
- metadata +17 -2
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "coverage"
|
|
4
5
|
require "fileutils"
|
|
6
|
+
require "pathname"
|
|
7
|
+
require "English"
|
|
8
|
+
require "tmpdir"
|
|
5
9
|
|
|
6
10
|
module Saboteur
|
|
7
11
|
module Frameworks
|
|
8
|
-
# Opt-in Rails support (`rails
|
|
12
|
+
# Opt-in Rails support (`c.rails = true` in config/saboteur.rb).
|
|
9
13
|
#
|
|
10
14
|
# Refuses to run unless RAILS_ENV is test and every configured database is
|
|
11
15
|
# local, eager loads the app so every subject's owner is defined before
|
|
@@ -14,9 +18,23 @@ module Saboteur
|
|
|
14
18
|
class Rails
|
|
15
19
|
LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
|
|
16
20
|
|
|
21
|
+
# How many blocking session names the template-copy error lists.
|
|
22
|
+
MAX_BLOCKER_NAMES = 5
|
|
23
|
+
|
|
24
|
+
# sqlite3-ruby's Backup#step returns raw SQLite status codes instead
|
|
25
|
+
# of raising, and finish discards its own: without an explicit check
|
|
26
|
+
# a partial destination would pass as a copy.
|
|
27
|
+
SQLITE_DONE = 101
|
|
28
|
+
SQLITE_BUSY = 5
|
|
29
|
+
SQLITE_LOCKED = 6
|
|
30
|
+
BACKUP_ATTEMPTS = 5
|
|
31
|
+
BACKUP_RETRY_DELAY = 0.05
|
|
32
|
+
|
|
17
33
|
def initialize(config, hooks)
|
|
18
34
|
@config = config
|
|
19
35
|
@hooks = hooks
|
|
36
|
+
@worker_copies = []
|
|
37
|
+
@memory_snapshots = {}
|
|
20
38
|
end
|
|
21
39
|
|
|
22
40
|
def before_boot
|
|
@@ -27,18 +45,31 @@ module Saboteur
|
|
|
27
45
|
# on macOS that goes through XPC, which crashes a forked process. The
|
|
28
46
|
# databases are local only (check_databases), so it buys nothing.
|
|
29
47
|
ENV["PGGSSENCMODE"] ||= "disable"
|
|
48
|
+
|
|
49
|
+
# Bytecode from Bootsnap's compile cache carries no coverage counters,
|
|
50
|
+
# so a file served from the cache never appears in Coverage.result.
|
|
51
|
+
# Turn the cache off for this process before config/boot loads
|
|
52
|
+
# Bootsnap; the load-path cache still works. Bootsnap checks
|
|
53
|
+
# DISABLE_<feature> by presence, so a value the user set disables the
|
|
54
|
+
# cache too. Files loaded during the eager load are also covered by
|
|
55
|
+
# eager_load_with_coverage_running.
|
|
56
|
+
ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] ||= "1"
|
|
30
57
|
end
|
|
31
58
|
|
|
32
59
|
def after_boot
|
|
33
60
|
unless defined?(::Rails.env)
|
|
34
61
|
raise Error,
|
|
35
|
-
"rails: true but Rails did not load (add config/environment to require
|
|
62
|
+
"rails: true but Rails did not load (add config/environment to require - for an engine, " \
|
|
63
|
+
"its dummy app's environment, such as test/dummy/config/environment.rb; only a dynamically " \
|
|
64
|
+
"created dummy app has no environment file, and there you require the helper that calls " \
|
|
65
|
+
"setup instead, such as DummyApp.setup in spec/rails_helper)"
|
|
36
66
|
end
|
|
37
67
|
raise Error, "rails integration refuses to run: Rails.env is #{::Rails.env}" unless ::Rails.env.test?
|
|
38
68
|
|
|
39
69
|
check_databases
|
|
40
|
-
|
|
70
|
+
eager_load_with_coverage_running
|
|
41
71
|
disconnect
|
|
72
|
+
snapshot_memory_databases
|
|
42
73
|
@hooks.register(:before_fork) { |_mutation| disconnect }
|
|
43
74
|
@hooks.register(:after_fork) { reconnect }
|
|
44
75
|
end
|
|
@@ -54,16 +85,58 @@ module Saboteur
|
|
|
54
85
|
end
|
|
55
86
|
|
|
56
87
|
# Parent, before workers start: one copy of each test database per
|
|
57
|
-
# extra worker, made from the prepared test database.
|
|
88
|
+
# extra worker, made from the prepared test database. Every attempted
|
|
89
|
+
# copy is tracked before it runs, so cleanup_workers drops them all,
|
|
90
|
+
# including the partial set when a copy failed.
|
|
58
91
|
def prepare_workers(jobs)
|
|
59
92
|
return unless defined?(::ActiveRecord::Base) && jobs > 1
|
|
60
93
|
|
|
94
|
+
@worker_copies = []
|
|
61
95
|
test_configs.each do |db|
|
|
62
|
-
|
|
96
|
+
# An in-memory SQLite database lives in the parent's connection;
|
|
97
|
+
# workers inherit it through fork, so there is nothing to copy.
|
|
98
|
+
next if self.class.memory_database?(db)
|
|
99
|
+
|
|
100
|
+
(1...jobs).each do |number|
|
|
101
|
+
name = self.class.worker_database(db.database, number)
|
|
102
|
+
# Track the target before the copy, not after it: an attempt
|
|
103
|
+
# that created the database and then failed (a post-CREATE
|
|
104
|
+
# error, a partial SQLite copy) must still be dropped at the
|
|
105
|
+
# end of the run. Dropping is idempotent (DROP IF EXISTS,
|
|
106
|
+
# rm -f), so tracking a copy that never happened costs nothing.
|
|
107
|
+
@worker_copies << [db, name]
|
|
108
|
+
copy_database(db, name)
|
|
109
|
+
end
|
|
63
110
|
end
|
|
64
111
|
disconnect
|
|
65
112
|
end
|
|
66
113
|
|
|
114
|
+
# Parent, after the run: drop every copy prepare_workers made, so runs
|
|
115
|
+
# do not collect one database per worker. Every copy is attempted even
|
|
116
|
+
# when a drop fails; the failures are reported together at the end.
|
|
117
|
+
def cleanup_workers
|
|
118
|
+
return if @worker_copies.empty? && @memory_snapshot_dir.nil?
|
|
119
|
+
|
|
120
|
+
failures = @worker_copies.filter_map do |db, name|
|
|
121
|
+
drop_database(db, name)
|
|
122
|
+
nil
|
|
123
|
+
rescue StandardError => e
|
|
124
|
+
"#{name}: #{e.message}"
|
|
125
|
+
end
|
|
126
|
+
@worker_copies = []
|
|
127
|
+
if @memory_snapshot_dir
|
|
128
|
+
FileUtils.rm_rf(@memory_snapshot_dir)
|
|
129
|
+
@memory_snapshot_dir = nil
|
|
130
|
+
@memory_snapshots = {}
|
|
131
|
+
end
|
|
132
|
+
begin
|
|
133
|
+
disconnect
|
|
134
|
+
rescue StandardError => e
|
|
135
|
+
failures << "disconnect: #{e.message}"
|
|
136
|
+
end
|
|
137
|
+
raise Error, "could not drop worker database copies: #{failures.join('; ')}" if failures.any?
|
|
138
|
+
end
|
|
139
|
+
|
|
67
140
|
# Worker: point every connection pool (primary, and any cache, queue or
|
|
68
141
|
# connects_to databases, for every role and shard) at this worker's
|
|
69
142
|
# copy of its database.
|
|
@@ -74,11 +147,16 @@ module Saboteur
|
|
|
74
147
|
handler.connection_pool_list(:all).to_a.each do |pool|
|
|
75
148
|
config = pool.pool_config
|
|
76
149
|
db = pool.db_config
|
|
150
|
+
# An in-memory SQLite database would be discarded by
|
|
151
|
+
# re-establishing: the worker keeps its own connection.
|
|
152
|
+
next if self.class.memory_database?(db)
|
|
153
|
+
|
|
77
154
|
handler.establish_connection(db.configuration_hash.merge(database: self.class.worker_database(db.database,
|
|
78
155
|
number)),
|
|
79
156
|
owner_name: self.class.owner_of(config), role: config.role,
|
|
80
157
|
shard: config.shard)
|
|
81
158
|
end
|
|
159
|
+
restore_memory_databases
|
|
82
160
|
end
|
|
83
161
|
|
|
84
162
|
def self.owner_of(pool_config)
|
|
@@ -87,12 +165,35 @@ module Saboteur
|
|
|
87
165
|
Object.const_get(pool_config.connection_descriptor.name)
|
|
88
166
|
end
|
|
89
167
|
|
|
168
|
+
# Only SQLite can be in-memory; every other adapter names a server.
|
|
169
|
+
def self.memory_database?(db)
|
|
170
|
+
db.adapter.to_s.match?(/sqlite/) && db.database.to_s == ":memory:"
|
|
171
|
+
end
|
|
172
|
+
|
|
90
173
|
def self.local_host?(host)
|
|
91
174
|
host.nil? || host.to_s.empty? || host.to_s.start_with?("/") || LOCAL_HOSTS.include?(host.to_s.downcase)
|
|
92
175
|
end
|
|
93
176
|
|
|
94
177
|
private
|
|
95
178
|
|
|
179
|
+
# Bootsnap before 1.25 skips its bytecode cache only while Coverage is
|
|
180
|
+
# running (1.25 and later skip it whenever Coverage is set up, suspended
|
|
181
|
+
# or not), and Coverage is suspended while the app boots (exe/saboteur
|
|
182
|
+
# sets it up suspended). Cached bytecode carries no coverage counters.
|
|
183
|
+
# Resume Coverage just for the eager load, so every app file compiles
|
|
184
|
+
# with counters. The coverage recorder clears the counts before the
|
|
185
|
+
# suite runs, so load-time hits are never credited to a test.
|
|
186
|
+
def eager_load_with_coverage_running
|
|
187
|
+
return ::Rails.application.eager_load! unless Coverage.state == :suspended
|
|
188
|
+
|
|
189
|
+
Coverage.resume
|
|
190
|
+
begin
|
|
191
|
+
::Rails.application.eager_load!
|
|
192
|
+
ensure
|
|
193
|
+
Coverage.suspend
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
96
197
|
def check_databases
|
|
97
198
|
return unless defined?(::ActiveRecord::Base)
|
|
98
199
|
|
|
@@ -118,29 +219,157 @@ module Saboteur
|
|
|
118
219
|
disconnect
|
|
119
220
|
copy_postgres(db, name)
|
|
120
221
|
when /sqlite/
|
|
121
|
-
|
|
122
|
-
["", "-wal", "-shm"].each do |suffix|
|
|
123
|
-
FileUtils.cp("#{db.database}#{suffix}", "#{name}#{suffix}") if File.exist?("#{db.database}#{suffix}")
|
|
124
|
-
end
|
|
222
|
+
copy_sqlite(db, name)
|
|
125
223
|
else
|
|
126
224
|
raise Error, "--jobs > 1 with Rails needs Postgres or SQLite (got #{db.adapter})"
|
|
127
225
|
end
|
|
128
226
|
end
|
|
129
227
|
|
|
228
|
+
# The mirror of copy_database. Adapters copy_database rejects never get
|
|
229
|
+
# copies, so there is nothing to drop for them.
|
|
230
|
+
def drop_database(db, name)
|
|
231
|
+
case db.adapter
|
|
232
|
+
when /postg/
|
|
233
|
+
disconnect
|
|
234
|
+
drop_postgres(db, name)
|
|
235
|
+
when /sqlite/
|
|
236
|
+
target = sqlite_absolute(name)
|
|
237
|
+
["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# A relative sqlite path means something only to Rails: 7.1+ resolves
|
|
242
|
+
# it against Rails.root, not the process working directory. Resolve
|
|
243
|
+
# the source the same way, checkpoint so a live WAL database holds
|
|
244
|
+
# every committed write in its main file, then snapshot it through
|
|
245
|
+
# the backup API. A missing source is an error, not a skip: otherwise
|
|
246
|
+
# the worker starts from a fresh empty database and silently tests
|
|
247
|
+
# nothing.
|
|
248
|
+
def copy_sqlite(db, name)
|
|
249
|
+
source = sqlite_absolute(db.database)
|
|
250
|
+
unless File.exist?(source)
|
|
251
|
+
raise Error, "test database #{source} does not exist; run the test suite once to create it"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
checkpoint_sqlite(source)
|
|
255
|
+
backup_sqlite(source, sqlite_absolute(name))
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def sqlite_absolute(database)
|
|
259
|
+
return database if Pathname.new(database).absolute?
|
|
260
|
+
return database unless defined?(::Rails)
|
|
261
|
+
|
|
262
|
+
File.expand_path(database, ::Rails.root)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# wal_checkpoint(TRUNCATE) reports instead of raising: [busy, frames
|
|
266
|
+
# in the WAL, frames checkpointed]. A busy reader or an incomplete
|
|
267
|
+
# pass leaves committed writes outside the main file, and copying on
|
|
268
|
+
# would silently miss them - fail closed instead. Through a
|
|
269
|
+
# connection of saboteur's own, so the app's connections never point
|
|
270
|
+
# at the database being checkpointed.
|
|
271
|
+
def checkpoint_sqlite(source)
|
|
272
|
+
with_sqlite(source) do |database|
|
|
273
|
+
busy, frames, checkpointed = database.execute("PRAGMA wal_checkpoint(TRUNCATE)").first
|
|
274
|
+
return if busy.zero? && frames == checkpointed
|
|
275
|
+
|
|
276
|
+
raise Error, "could not checkpoint #{source} before copying it " \
|
|
277
|
+
"(busy: #{busy}, frames: #{frames}, checkpointed: #{checkpointed}); " \
|
|
278
|
+
"close the connections holding it and run again"
|
|
279
|
+
end
|
|
280
|
+
rescue SQLite3::BusyException => e
|
|
281
|
+
raise Error, "could not checkpoint #{source} before copying it (#{e.message}); " \
|
|
282
|
+
"close the connections holding it and run again"
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# A byte copy of main + WAL + SHM is no snapshot: a write landing
|
|
286
|
+
# between the file copies leaves the set malformed. The backup API
|
|
287
|
+
# copies a consistent image and restarts when the source changes
|
|
288
|
+
# under it. The target starts clean so stale sidecars from an
|
|
289
|
+
# earlier byte copy cannot shadow the fresh image.
|
|
290
|
+
def backup_sqlite(source, target)
|
|
291
|
+
["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
|
|
292
|
+
with_sqlite(source, readonly: true) do |src|
|
|
293
|
+
with_sqlite(target) do |dst|
|
|
294
|
+
backup = SQLite3::Backup.new(dst, "main", src, "main")
|
|
295
|
+
begin
|
|
296
|
+
step_backup(backup, target)
|
|
297
|
+
ensure
|
|
298
|
+
backup.finish
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# BUSY and LOCKED earn a bounded retry; DONE means the target holds
|
|
305
|
+
# every page; anything else fails the copy.
|
|
306
|
+
def step_backup(backup, target)
|
|
307
|
+
BACKUP_ATTEMPTS.times do
|
|
308
|
+
case (status = backup.step(-1))
|
|
309
|
+
when SQLITE_DONE then return
|
|
310
|
+
when SQLITE_BUSY, SQLITE_LOCKED
|
|
311
|
+
sleep(BACKUP_RETRY_DELAY)
|
|
312
|
+
else
|
|
313
|
+
raise Error, "could not copy the database to #{target}: sqlite backup failed with status #{status}"
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
raise Error, "could not copy the database to #{target}: still busy after #{BACKUP_ATTEMPTS} attempts"
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def with_sqlite(path, **)
|
|
320
|
+
require "sqlite3"
|
|
321
|
+
|
|
322
|
+
database = SQLite3::Database.new(path, **)
|
|
323
|
+
begin
|
|
324
|
+
yield database
|
|
325
|
+
ensure
|
|
326
|
+
database.close
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
130
330
|
# Through a connection class of saboteur's own, so the app's
|
|
131
331
|
# connections never point at the maintenance database.
|
|
132
332
|
def copy_postgres(db, name)
|
|
133
|
-
|
|
134
|
-
admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
|
|
135
|
-
admin.connection_pool.with_connection do |connection|
|
|
333
|
+
with_maintenance_connection(db) do |connection|
|
|
136
334
|
connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
|
|
335
|
+
refuse_busy_template(connection, db.database)
|
|
137
336
|
connection.execute("CREATE DATABASE #{connection.quote_table_name(name)} " \
|
|
138
337
|
"TEMPLATE #{connection.quote_table_name(db.database)}")
|
|
139
338
|
end
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def drop_postgres(db, name)
|
|
342
|
+
with_maintenance_connection(db) do |connection|
|
|
343
|
+
connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Through a connection class of saboteur's own, so the app's
|
|
348
|
+
# connections never point at the maintenance database.
|
|
349
|
+
def with_maintenance_connection(db, &)
|
|
350
|
+
admin = maintenance_record
|
|
351
|
+
admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
|
|
352
|
+
admin.connection_pool.with_connection(&)
|
|
140
353
|
ensure
|
|
141
354
|
admin&.remove_connection
|
|
142
355
|
end
|
|
143
356
|
|
|
357
|
+
# Postgres refuses CREATE DATABASE ... TEMPLATE while any other session
|
|
358
|
+
# is connected to the template, so name the blockers instead of dying
|
|
359
|
+
# mid-run with the raw error. Never terminates other sessions.
|
|
360
|
+
def refuse_busy_template(connection, database)
|
|
361
|
+
blockers = connection.select_values(<<~SQL)
|
|
362
|
+
SELECT application_name FROM pg_stat_activity
|
|
363
|
+
WHERE datname = #{connection.quote(database)} AND pid <> pg_backend_pid()
|
|
364
|
+
SQL
|
|
365
|
+
return if blockers.empty?
|
|
366
|
+
|
|
367
|
+
names = blockers.map { |app| app.to_s.empty? ? "unnamed" : app }.first(MAX_BLOCKER_NAMES)
|
|
368
|
+
raise Error, "cannot copy the test database #{database}: #{blockers.size} other session(s) connected " \
|
|
369
|
+
"(#{names.join(', ')})#{', ...' if blockers.size > names.size}. " \
|
|
370
|
+
"Stop other test runs, consoles or Spring, then run saboteur again."
|
|
371
|
+
end
|
|
372
|
+
|
|
144
373
|
# ActiveRecord names a connection's owner by class name, so the class
|
|
145
374
|
# needs a constant.
|
|
146
375
|
def maintenance_record
|
|
@@ -149,10 +378,15 @@ module Saboteur
|
|
|
149
378
|
self.class.const_set(:MaintenanceRecord, Class.new(::ActiveRecord::Base) { self.abstract_class = true })
|
|
150
379
|
end
|
|
151
380
|
|
|
381
|
+
# Disconnecting an in-memory SQLite pool destroys the database, so
|
|
382
|
+
# those pools stay connected everywhere saboteur disconnects: the
|
|
383
|
+
# schema loaded at boot survives, and workers inherit it via fork.
|
|
152
384
|
def disconnect
|
|
153
385
|
return unless defined?(::ActiveRecord::Base)
|
|
154
386
|
|
|
155
|
-
::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each
|
|
387
|
+
::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
|
|
388
|
+
pool.disconnect! unless self.class.memory_database?(pool.db_config)
|
|
389
|
+
end
|
|
156
390
|
end
|
|
157
391
|
|
|
158
392
|
# The parent disconnected before fork, so inherited pools hold no sockets.
|
|
@@ -160,6 +394,88 @@ module Saboteur
|
|
|
160
394
|
# checks out a fresh connection of its own on first use.
|
|
161
395
|
def reconnect
|
|
162
396
|
disconnect
|
|
397
|
+
restore_memory_databases
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Parent, once the app booted: an in-memory SQLite database survives
|
|
401
|
+
# only inside its own connection, and a forked child cannot keep it -
|
|
402
|
+
# ActiveRecord discards inherited pools and the sqlite3 gem closes
|
|
403
|
+
# writable handles in the child, so the next access there opens an
|
|
404
|
+
# empty database. Snapshot each in-memory database to a file the
|
|
405
|
+
# children can restore from.
|
|
406
|
+
def snapshot_memory_databases
|
|
407
|
+
return unless defined?(::ActiveRecord::Base)
|
|
408
|
+
|
|
409
|
+
pools = ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
|
|
410
|
+
memory_pools = pools.select { |pool| self.class.memory_database?(pool.db_config) }
|
|
411
|
+
return if memory_pools.empty?
|
|
412
|
+
|
|
413
|
+
@memory_snapshot_dir = Dir.mktmpdir("saboteur-memory-")
|
|
414
|
+
own_memory_snapshots
|
|
415
|
+
memory_pools.each_with_index do |pool, index|
|
|
416
|
+
path = File.join(@memory_snapshot_dir, "#{index}.sqlite3")
|
|
417
|
+
pool.with_connection do |connection|
|
|
418
|
+
file = SQLite3::Database.new(path)
|
|
419
|
+
with_backup(file, connection.raw_connection, close: file) { |backup| step_backup(backup, path) }
|
|
420
|
+
end
|
|
421
|
+
@memory_snapshots[pool.db_config] = path
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# The safety net for an abrupt process exit: Engine.call removes the
|
|
426
|
+
# snapshots at the run boundary on every normal path, so this handler
|
|
427
|
+
# fires only when the owner process dies first. Forked children
|
|
428
|
+
# inherit the handler but not the pid.
|
|
429
|
+
def own_memory_snapshots
|
|
430
|
+
owner = Process.pid
|
|
431
|
+
at_exit { cleanup_memory_snapshots(owner) }
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def cleanup_memory_snapshots(owner)
|
|
435
|
+
FileUtils.rm_rf(@memory_snapshot_dir) if Process.pid == owner && @memory_snapshot_dir
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Yields a backup and always returns both handles: exactly one finish
|
|
439
|
+
# per init (skipped only when backup_init itself raised, leaving no
|
|
440
|
+
# handle), and the database closes even when finish fails.
|
|
441
|
+
def with_backup(destination, source, close:)
|
|
442
|
+
backup = SQLite3::Backup.new(destination, "main", source, "main")
|
|
443
|
+
yield backup
|
|
444
|
+
ensure
|
|
445
|
+
begin
|
|
446
|
+
backup&.finish
|
|
447
|
+
ensure
|
|
448
|
+
close_without_masking(close)
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# A close can fail too, and a close that fails on the way out must
|
|
453
|
+
# not hide the error already in flight.
|
|
454
|
+
def close_without_masking(database)
|
|
455
|
+
in_flight = $ERROR_INFO
|
|
456
|
+
database.close
|
|
457
|
+
rescue SQLite3::Exception
|
|
458
|
+
raise unless in_flight
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# Child: load the parent's snapshot into this process's own fresh
|
|
462
|
+
# in-memory database. The backup overwrites the destination, so
|
|
463
|
+
# restoring more than once is harmless.
|
|
464
|
+
def restore_memory_databases
|
|
465
|
+
return unless defined?(::ActiveRecord::Base)
|
|
466
|
+
return if @memory_snapshots.empty?
|
|
467
|
+
|
|
468
|
+
::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
|
|
469
|
+
snapshot = @memory_snapshots[pool.db_config]
|
|
470
|
+
next unless snapshot
|
|
471
|
+
|
|
472
|
+
pool.with_connection do |connection|
|
|
473
|
+
source = SQLite3::Database.new(snapshot, readonly: true)
|
|
474
|
+
with_backup(connection.raw_connection, source, close: source) do |backup|
|
|
475
|
+
step_backup(backup, pool.db_config.database)
|
|
476
|
+
end
|
|
477
|
+
end
|
|
478
|
+
end
|
|
163
479
|
end
|
|
164
480
|
end
|
|
165
481
|
end
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
-
require "json"
|
|
5
4
|
require_relative "config"
|
|
6
5
|
|
|
7
6
|
module Saboteur
|
|
@@ -18,20 +17,20 @@ module Saboteur
|
|
|
18
17
|
DOCUMENTATION = "https://github.com/vamsipavanmahesh/saboteur"
|
|
19
18
|
|
|
20
19
|
EXIT_CODE_MEANINGS = {
|
|
21
|
-
0 => "Success. Without --threshold: no unignored survivors
|
|
22
|
-
"it, even if mutants survived.",
|
|
20
|
+
0 => "Success. Without --threshold: no unignored survivors, and at most half of the executed mutants " \
|
|
21
|
+
"errored. With --threshold: the score is at or above it, even if mutants survived.",
|
|
23
22
|
1 => "Without --threshold: at least one mutant survived and is not on the ignore list. With --threshold " \
|
|
24
23
|
"(survivors alone no longer fail the run): the score is below it, or no mutant was tested so " \
|
|
25
24
|
"there is no score.",
|
|
26
|
-
2 => "Usage or configuration error: fix the command or
|
|
25
|
+
2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
|
|
27
26
|
"mutant is only found after the target has booted. A named source or test file that does not exist.",
|
|
28
|
-
3 => "Tool error: saboteur could not finish, or
|
|
29
|
-
"also any errored or resource-limited mutant."
|
|
27
|
+
3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
|
|
28
|
+
"a run tested almost nothing). With --threshold, also any errored or resource-limited mutant."
|
|
30
29
|
}.freeze
|
|
31
30
|
|
|
32
31
|
DOCTOR_EXIT_CODE_MEANINGS = {
|
|
33
32
|
0 => "Healthy: the environment checks passed and the baseline passed twice in the same order.",
|
|
34
|
-
2 => "Usage or configuration error: fix the command or
|
|
33
|
+
2 => "Usage or configuration error: fix the command or config/saboteur.rb.",
|
|
35
34
|
3 => "A check failed: the target did not boot, or the baseline failed or changed between runs."
|
|
36
35
|
}.freeze
|
|
37
36
|
|
|
@@ -53,7 +52,8 @@ module Saboteur
|
|
|
53
52
|
["Replay one mutant by the digest printed next to a survivor", "saboteur run --id 1a2b3c4d5e6f7a8b"],
|
|
54
53
|
["Mutate named files against named specs, skipping the whole-suite coverage run",
|
|
55
54
|
"saboteur run lib/a.rb lib/b.rb --test spec/a_spec.rb --test spec/b_spec.rb"],
|
|
56
|
-
["CI gate: exit 1 when the score is below 90 percent", "saboteur run --threshold 90"]
|
|
55
|
+
["CI gate: exit 1 when the score is below 90 percent", "saboteur run --threshold 90"],
|
|
56
|
+
["AI agent: bounded JSON summary and strict exit codes", "saboteur run --agent --since origin/main"]
|
|
57
57
|
]
|
|
58
58
|
),
|
|
59
59
|
Command.new(
|
|
@@ -65,7 +65,7 @@ module Saboteur
|
|
|
65
65
|
takes_options: true,
|
|
66
66
|
exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
|
|
67
67
|
examples: [["Check the setup before the first run", "saboteur doctor"],
|
|
68
|
-
["Use another config file and seed", "saboteur doctor --config ci.
|
|
68
|
+
["Use another config file and seed", "saboteur doctor --config config/ci.rb --seed 7"]]
|
|
69
69
|
),
|
|
70
70
|
Command.new(
|
|
71
71
|
name: "version",
|
|
@@ -83,12 +83,25 @@ module Saboteur
|
|
|
83
83
|
takes_options: false,
|
|
84
84
|
exit_codes: nil,
|
|
85
85
|
examples: [["Show everything", "saboteur help"], ["Show one command", "saboteur help run"]]
|
|
86
|
+
),
|
|
87
|
+
Command.new(
|
|
88
|
+
name: "agent-skill",
|
|
89
|
+
summary: "Print instructions for AI coding agents to paste into AGENTS.md or CLAUDE.md",
|
|
90
|
+
description: "Prints a short Markdown block that tells an agent when to run saboteur, how to read " \
|
|
91
|
+
"`saboteur run --agent`, and how to handle survivors.",
|
|
92
|
+
takes_options: false,
|
|
93
|
+
exit_codes: nil,
|
|
94
|
+
examples: [["Add the agent instructions to your repo", "saboteur agent-skill >> AGENTS.md"]]
|
|
86
95
|
)
|
|
87
96
|
].freeze
|
|
88
97
|
|
|
98
|
+
AGENT_EXIT_CODES = " With --agent the codes are stricter: 0 only when the run was complete and clean; " \
|
|
99
|
+
"1 survivors to review;\n 2 usage or config error; 3 incomplete (no coverage, caps, errors, " \
|
|
100
|
+
"nothing selected) or tool failure."
|
|
101
|
+
|
|
89
102
|
FILES = [
|
|
90
|
-
["
|
|
91
|
-
[".
|
|
103
|
+
["config/saboteur.rb", "Settings as Ruby (Saboteur.configure). Command line options override it."],
|
|
104
|
+
["config/saboteur_ignore.rb", "Accepted survivors: mutation digests with subject and a required reason."],
|
|
92
105
|
[".saboteur/report.json", "Stryker mutation-testing-report-schema JSON (--json changes the path)."],
|
|
93
106
|
[".saboteur/report.html", "Self-contained HTML report (--html changes the path)."],
|
|
94
107
|
[".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete."]
|
|
@@ -169,18 +182,21 @@ module Saboteur
|
|
|
169
182
|
def exit_codes_section(codes, command_name = nil)
|
|
170
183
|
rows = codes.map { |code, meaning| " #{code} #{meaning}" }
|
|
171
184
|
title = command_name ? "EXIT CODES (saboteur #{command_name})" : "EXIT CODES"
|
|
172
|
-
"#{
|
|
185
|
+
note = codes.equal?(EXIT_CODE_MEANINGS) ? "#{AGENT_EXIT_CODES}\n" : ""
|
|
186
|
+
"#{title}\n#{rows.join("\n")}\n#{note}"
|
|
173
187
|
end
|
|
174
188
|
|
|
175
189
|
def config_section
|
|
176
|
-
rows = Config::DEFAULTS.map { |key, value| "
|
|
177
|
-
intro = "Every
|
|
178
|
-
"CONFIG FILE (
|
|
190
|
+
rows = Config::DEFAULTS.map { |key, value| " c.#{key} = #{describe_default(value)}" }
|
|
191
|
+
intro = "Every setting with its default value, as a Ruby assignment. Unknown settings are an error."
|
|
192
|
+
"CONFIG FILE (config/saboteur.rb)\n#{intro}\n#{rows.join("\n")}\n"
|
|
179
193
|
end
|
|
180
194
|
|
|
181
|
-
#
|
|
195
|
+
# Ruby literals, so a copied line loads as the same value: inspect prints
|
|
196
|
+
# nil as "nil" (JSON would print null, which is not Ruby) and quotes
|
|
197
|
+
# strings and arrays the way Ruby reads them back.
|
|
182
198
|
def describe_default(value)
|
|
183
|
-
|
|
199
|
+
value.inspect
|
|
184
200
|
end
|
|
185
201
|
|
|
186
202
|
def files_section
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -1,40 +1,94 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
-
require "yaml"
|
|
5
|
-
|
|
6
4
|
module Saboteur
|
|
7
|
-
#
|
|
5
|
+
# Entry point used by config/saboteur_ignore.rb.
|
|
6
|
+
def self.ignore
|
|
7
|
+
collector = IgnoreList.current_collector
|
|
8
|
+
raise "Saboteur.ignore is only valid while config/saboteur_ignore.rb loads" unless collector
|
|
9
|
+
|
|
10
|
+
yield collector
|
|
11
|
+
collector
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Survivors the team has accepted. config/saboteur_ignore.rb declares them:
|
|
8
15
|
#
|
|
9
|
-
#
|
|
10
|
-
# mutation
|
|
11
|
-
#
|
|
16
|
+
# Saboteur.ignore do |i|
|
|
17
|
+
# i.mutation "1a2b3c4d5e6f7a8b",
|
|
18
|
+
# subject: "Person#adult?",
|
|
19
|
+
# reason: "equivalent: age is always an integer"
|
|
20
|
+
# end
|
|
12
21
|
#
|
|
13
22
|
# reason is mandatory; a file with a missing or blank reason does not load.
|
|
14
23
|
class IgnoreList
|
|
15
24
|
Entry = Data.define(:subject, :mutation, :reason)
|
|
16
25
|
|
|
26
|
+
# Collects the entries a saboteur_ignore.rb file declares.
|
|
27
|
+
class Collector
|
|
28
|
+
attr_reader :entries
|
|
29
|
+
|
|
30
|
+
def initialize
|
|
31
|
+
@entries = []
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def mutation(digest = nil, subject: nil, reason: nil)
|
|
35
|
+
line = caller_locations(1, 1).first.lineno
|
|
36
|
+
values = { "subject" => subject, "mutation" => digest, "reason" => reason }
|
|
37
|
+
missing = values.select { |_, value| value.to_s.strip.empty? }.keys
|
|
38
|
+
raise ConfigError, "#{line}: entry is missing #{missing.join(', ')}" if missing.any?
|
|
39
|
+
|
|
40
|
+
@entries << Entry.new(
|
|
41
|
+
subject: subject.to_s.strip,
|
|
42
|
+
mutation: digest.to_s.strip,
|
|
43
|
+
reason: reason.to_s.strip
|
|
44
|
+
)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
17
48
|
attr_reader :entries
|
|
18
49
|
|
|
50
|
+
# The collector is process-global on purpose: the ignore file calls the
|
|
51
|
+
# top-level Saboteur.ignore, which must reach the load in progress.
|
|
52
|
+
# Loads are therefore serialized - never load two ignore files at once.
|
|
19
53
|
def self.load(path)
|
|
20
54
|
return new([]) unless path && File.file?(path)
|
|
21
55
|
|
|
22
|
-
|
|
23
|
-
|
|
56
|
+
@collector = Collector.new
|
|
57
|
+
Kernel.load(path)
|
|
58
|
+
new(@collector.entries)
|
|
59
|
+
rescue ConfigError => e
|
|
60
|
+
raise ConfigError, "#{path}:#{e.message}"
|
|
61
|
+
rescue NoMethodError => e
|
|
62
|
+
raise ConfigError, unknown_directive_message(path, e) if e.receiver.is_a?(Collector)
|
|
24
63
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
raise ConfigError,
|
|
64
|
+
# A typo'd call outside the collector: same treatment as a bare
|
|
65
|
+
# constant, or the raw NoMethodError points at our internals.
|
|
66
|
+
raise ConfigError, name_error_message(path, e)
|
|
67
|
+
rescue SyntaxError, LoadError => e
|
|
68
|
+
raise ConfigError, "#{path}: #{e.message.lines.first.to_s.chomp}"
|
|
69
|
+
rescue NameError => e
|
|
70
|
+
raise ConfigError, name_error_message(path, e)
|
|
71
|
+
ensure
|
|
72
|
+
@collector = nil
|
|
28
73
|
end
|
|
29
74
|
|
|
30
|
-
def self.
|
|
31
|
-
|
|
75
|
+
def self.unknown_directive_message(path, error)
|
|
76
|
+
line = error.backtrace_locations.find { |loc| loc.path == path }&.lineno
|
|
77
|
+
"#{path}:#{line}: unknown directive #{error.name.to_s.inspect} - " \
|
|
78
|
+
'declare entries with mutation "<digest>", subject:, reason:'
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self.name_error_message(path, error)
|
|
82
|
+
line = error.backtrace_locations&.find { |loc| loc.path == path }&.lineno
|
|
83
|
+
"#{path}:#{line}: #{error.message.lines.first.to_s.chomp} " \
|
|
84
|
+
"- this file loads before the app boots; use plain Ruby or ENV"
|
|
85
|
+
end
|
|
32
86
|
|
|
33
|
-
|
|
34
|
-
missing = values.select { |_, value| value.empty? }.keys
|
|
35
|
-
raise ConfigError, "#{path}: entry #{index + 1} is missing #{missing.join(', ')}" if missing.any?
|
|
87
|
+
private_class_method :unknown_directive_message, :name_error_message
|
|
36
88
|
|
|
37
|
-
|
|
89
|
+
# The Collector Saboteur.ignore hands to the file being loaded.
|
|
90
|
+
def self.current_collector
|
|
91
|
+
@collector
|
|
38
92
|
end
|
|
39
93
|
|
|
40
94
|
def initialize(entries)
|