inferno_core 1.4.1 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/inferno/apps/cli/execute_script.rb +22 -4
- data/lib/inferno/apps/cli/session/session_compare.rb +27 -3
- data/lib/inferno/apps/cli/session/session_status.rb +20 -4
- data/lib/inferno/apps/web/controllers/test_runs/create.rb +8 -1
- data/lib/inferno/apps/web/controllers/test_runs/destroy.rb +10 -1
- data/lib/inferno/config/boot/db.rb +45 -0
- data/lib/inferno/config/boot/executor.rb +4 -0
- data/lib/inferno/config/boot/web.rb +4 -0
- data/lib/inferno/dsl/input_output_handling.rb +7 -5
- data/lib/inferno/dsl/resume_test_route.rb +10 -1
- data/lib/inferno/dsl/suite_endpoint.rb +12 -1
- data/lib/inferno/jobs.rb +5 -1
- data/lib/inferno/test_runner.rb +35 -17
- data/lib/inferno/utils/db_concurrency_check.rb +166 -0
- data/lib/inferno/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f1318358aa5a9dfd3f856740eb3b02ca30c5a960893595e44631fea414906936
|
|
4
|
+
data.tar.gz: ddf3a22b024d5745efeead73eacc55a23d7c0b9841e57177df877ed46457dc3a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5ddfa620841f6acb416190f683235fefeb089e623f238578ac7cd0022bc01ece6aee0644bae93b6d9f4ee72aff33fa4cefeda871a9a160502cd26dde0b193260
|
|
7
|
+
data.tar.gz: 718dcda3cd89f1f9d673eaa7596ec5dc8b4dd405b9434c6a55eb2caf381d2cd1516614098ac565d1104c410991ec6496b5e11ecfaf66df77b4d49df1ae5c569f
|
|
@@ -137,7 +137,7 @@ module Inferno
|
|
|
137
137
|
)
|
|
138
138
|
|
|
139
139
|
ExecutionStatus = Struct.new(
|
|
140
|
-
:done, :failed, :timed_out, :current_session, :current_timeout, :last_log_time,
|
|
140
|
+
:done, :failed, :timed_out, :cancel_pending, :current_session, :current_timeout, :last_log_time,
|
|
141
141
|
:cross_session_status, :last_step_signatures
|
|
142
142
|
)
|
|
143
143
|
|
|
@@ -152,6 +152,7 @@ module Inferno
|
|
|
152
152
|
done: false,
|
|
153
153
|
failed: false,
|
|
154
154
|
timed_out: false,
|
|
155
|
+
cancel_pending: false,
|
|
155
156
|
current_session: sessions.first,
|
|
156
157
|
current_timeout: options[:default_poll_timeout],
|
|
157
158
|
cross_session_status: {},
|
|
@@ -234,13 +235,15 @@ module Inferno
|
|
|
234
235
|
creator = Session::CreateSession.new(suite, session_create_options(session_config))
|
|
235
236
|
session_details = creator.create_session
|
|
236
237
|
key = session_config['name'] || suite
|
|
237
|
-
|
|
238
|
-
ScriptSession.new(
|
|
238
|
+
script_session = ScriptSession.new(
|
|
239
239
|
key: key,
|
|
240
240
|
suite_id: session_details['test_suite_id'],
|
|
241
241
|
session_id: session_details['id'],
|
|
242
242
|
short_id_map: extract_short_ids_from_session_details(session_details)
|
|
243
243
|
)
|
|
244
|
+
warn "Session created: #{session_details['id']}"
|
|
245
|
+
warn " Available at #{session_display_url(script_session)}"
|
|
246
|
+
script_session
|
|
244
247
|
end
|
|
245
248
|
|
|
246
249
|
def session_create_options(session_config)
|
|
@@ -442,6 +445,10 @@ module Inferno
|
|
|
442
445
|
|
|
443
446
|
# Returns a step hash to act on, or nil to keep polling.
|
|
444
447
|
def handle_actionable_status(status, session, timeout)
|
|
448
|
+
# A cancelled run's done status is indistinguishable from the
|
|
449
|
+
# runnable completing normally, so don't match steps against it.
|
|
450
|
+
return handle_cancel_completion(status, session, timeout) if execution_status.cancel_pending
|
|
451
|
+
|
|
445
452
|
matched_step = match_step(status, session.key)
|
|
446
453
|
|
|
447
454
|
if matched_step
|
|
@@ -463,10 +470,16 @@ module Inferno
|
|
|
463
470
|
last_completed = format_last_completed(last_completed_from_status(status), session.key)
|
|
464
471
|
warn "UNHANDLED WAIT - Canceling: session=#{session.key} last_completed=#{last_completed}"
|
|
465
472
|
execution_status.failed = true
|
|
473
|
+
execution_status.cancel_pending = true
|
|
466
474
|
attempt_cancel(session.session_id, status)
|
|
467
475
|
nil
|
|
468
476
|
end
|
|
469
477
|
|
|
478
|
+
def handle_cancel_completion(status, session, timeout)
|
|
479
|
+
warn "Cancellation complete: session=#{session.key} status=#{status['status']}"
|
|
480
|
+
{ command: nil, timeout: timeout, next_poll_session: nil }
|
|
481
|
+
end
|
|
482
|
+
|
|
470
483
|
def handle_unmatched_status(status, session, timeout)
|
|
471
484
|
run_status = status['status']
|
|
472
485
|
last_completed = format_last_completed(last_completed_from_status(status), session.key)
|
|
@@ -485,6 +498,7 @@ module Inferno
|
|
|
485
498
|
execution_status.failed = true
|
|
486
499
|
if run_status == 'waiting'
|
|
487
500
|
warn "Loop detected - Canceling: session=#{session.key} last_completed=#{last_completed}"
|
|
501
|
+
execution_status.cancel_pending = true
|
|
488
502
|
attempt_cancel(session.session_id, status)
|
|
489
503
|
return nil
|
|
490
504
|
else
|
|
@@ -504,10 +518,14 @@ module Inferno
|
|
|
504
518
|
last_completed = last_completed_from_status(status)
|
|
505
519
|
poll_status_last_test =
|
|
506
520
|
last_completed.present? ? " - last test: #{format_last_completed(last_completed, session_key)}" : ''
|
|
507
|
-
warn " [#{session_key}] #{status['status']}#{poll_status_last_test}"
|
|
521
|
+
warn " [#{session_key}] #{status['status']} (#{test_progress(status)})#{poll_status_last_test}"
|
|
508
522
|
execution_status.last_log_time = Time.now
|
|
509
523
|
end
|
|
510
524
|
|
|
525
|
+
def test_progress(status)
|
|
526
|
+
"#{status['completed_test_count']}/#{status['test_count']} tests"
|
|
527
|
+
end
|
|
528
|
+
|
|
511
529
|
def fetch_session_status(session_id)
|
|
512
530
|
Session::SessionStatus.new(session_id, options).status_for_session
|
|
513
531
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'csv'
|
|
2
2
|
require 'cgi'
|
|
3
|
+
require 'uri'
|
|
3
4
|
require_relative 'session_details'
|
|
4
5
|
require_relative 'session_results'
|
|
5
6
|
|
|
@@ -82,16 +83,39 @@ module Inferno
|
|
|
82
83
|
end
|
|
83
84
|
|
|
84
85
|
def save_actual_results_to_file
|
|
85
|
-
actual_results_file_name = "#{output_file_prefix}actual_results_#{results_timestamp}.json"
|
|
86
|
-
File.write(File.join(output_directory, actual_results_file_name), session_results
|
|
86
|
+
actual_results_file_name = "#{output_file_prefix}actual_results_#{results_timestamp}#{host_suffix}.json"
|
|
87
|
+
File.write(File.join(output_directory, actual_results_file_name), JSON.pretty_generate(session_results))
|
|
87
88
|
end
|
|
88
89
|
|
|
89
90
|
def save_comparison_csv_to_file
|
|
90
|
-
compared_csv_file_name = "#{output_file_prefix}compared_results_#{results_timestamp}.csv"
|
|
91
|
+
compared_csv_file_name = "#{output_file_prefix}compared_results_#{results_timestamp}#{host_suffix}.csv"
|
|
91
92
|
File.write(File.join(output_directory, compared_csv_file_name),
|
|
92
93
|
compared_results_as_csv)
|
|
93
94
|
end
|
|
94
95
|
|
|
96
|
+
# Filesystem-safe "_<host>" (plus non-default port, if any) suffix derived from
|
|
97
|
+
# the --inferno-base-url (-I) option, so output filenames can be disambiguated
|
|
98
|
+
# by target server. Empty when -I was not passed or the URL has no host.
|
|
99
|
+
def host_suffix
|
|
100
|
+
inferno_host_slug ? "_#{inferno_host_slug}" : ''
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def inferno_host_slug
|
|
104
|
+
return nil unless options[:inferno_base_url].present?
|
|
105
|
+
|
|
106
|
+
uri = begin
|
|
107
|
+
URI.parse(options[:inferno_base_url])
|
|
108
|
+
rescue URI::InvalidURIError
|
|
109
|
+
nil
|
|
110
|
+
end
|
|
111
|
+
host = uri&.host
|
|
112
|
+
return nil unless host
|
|
113
|
+
|
|
114
|
+
slug = host.dup
|
|
115
|
+
slug << "_#{uri.port}" if uri.port && uri.port != uri.default_port
|
|
116
|
+
slug.gsub(/[^a-zA-Z0-9.-]/, '_')
|
|
117
|
+
end
|
|
118
|
+
|
|
95
119
|
def display_compared_results
|
|
96
120
|
output = {
|
|
97
121
|
matched: results_match?,
|
|
@@ -27,7 +27,10 @@ module Inferno
|
|
|
27
27
|
|
|
28
28
|
if session_status['id'].present?
|
|
29
29
|
run_id = session_status['id']
|
|
30
|
-
|
|
30
|
+
test_results = run_results(run_id).select { |result| result['test_id'].present? }
|
|
31
|
+
session_status['completed_test_count'] = test_results.size
|
|
32
|
+
|
|
33
|
+
last_test_executed = last_test_executed(test_results, session_status['status'])
|
|
31
34
|
if last_test_executed.present?
|
|
32
35
|
session_status['last_test_executed'] = last_test_executed['test_id']
|
|
33
36
|
if session_status['status'] == 'waiting'
|
|
@@ -53,9 +56,22 @@ module Inferno
|
|
|
53
56
|
}
|
|
54
57
|
end
|
|
55
58
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
# Serialized updated_at values can collide when results are written in
|
|
60
|
+
# quick succession, so when the run is waiting, identify the waiting
|
|
61
|
+
# test by its 'wait' result rather than by timestamp order alone.
|
|
62
|
+
def last_test_executed(test_results, run_status)
|
|
63
|
+
if run_status == 'waiting'
|
|
64
|
+
wait_results = test_results.select { |result| result['result'] == 'wait' }
|
|
65
|
+
test_results = wait_results if wait_results.any?
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
most_recent_result(test_results)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# updated_at ties are broken by array position (later results win)
|
|
72
|
+
# since the API returns results in insertion order.
|
|
73
|
+
def most_recent_result(results)
|
|
74
|
+
results.each_with_index.max_by { |result, index| [result['updated_at'].to_s, index] }&.first
|
|
59
75
|
end
|
|
60
76
|
|
|
61
77
|
def run_results(run_id)
|
|
@@ -37,7 +37,14 @@ module Inferno
|
|
|
37
37
|
|
|
38
38
|
persist_inputs(session_data_repo, req.params, test_run.runnable)
|
|
39
39
|
|
|
40
|
-
Jobs.perform(
|
|
40
|
+
Jobs.perform(
|
|
41
|
+
Jobs::ExecuteTestRun,
|
|
42
|
+
test_run.id,
|
|
43
|
+
tags: [
|
|
44
|
+
"session:#{test_session.id}",
|
|
45
|
+
"run:#{test_run.test_suite_id || test_run.test_group_id || test_run.test_id}"
|
|
46
|
+
]
|
|
47
|
+
)
|
|
41
48
|
rescue Sequel::ValidationFailed, Sequel::ForeignKeyConstraintViolation,
|
|
42
49
|
Inferno::Exceptions::RequiredInputsNotFound,
|
|
43
50
|
Inferno::Exceptions::NotUserRunnableException => e
|
|
@@ -22,7 +22,16 @@ module Inferno
|
|
|
22
22
|
if test_run_is_waiting
|
|
23
23
|
waiting_result = results_repo.find_waiting_result(test_run_id: test_run.id)
|
|
24
24
|
results_repo.update_result(waiting_result.id, 'cancel', 'Test cancelled by user')
|
|
25
|
-
Jobs.perform(
|
|
25
|
+
Jobs.perform(
|
|
26
|
+
Jobs::ResumeTestRun,
|
|
27
|
+
test_run.id,
|
|
28
|
+
tags: [
|
|
29
|
+
'source:delete',
|
|
30
|
+
"session:#{test_run.test_session_id}",
|
|
31
|
+
"run:#{test_run.test_suite_id || test_run.test_group_id || test_run.test_id}",
|
|
32
|
+
"test:#{waiting_result.test_id}"
|
|
33
|
+
]
|
|
34
|
+
)
|
|
26
35
|
end
|
|
27
36
|
|
|
28
37
|
res.status = 204
|
|
@@ -1,6 +1,46 @@
|
|
|
1
1
|
require 'sequel'
|
|
2
2
|
require 'erb'
|
|
3
3
|
|
|
4
|
+
module Inferno
|
|
5
|
+
module Config
|
|
6
|
+
module Boot
|
|
7
|
+
# Extracted from the :db provider below so the sqlite-only branching can
|
|
8
|
+
# be unit tested with plain config hashes/doubles, instead of requiring
|
|
9
|
+
# a real postgres connection (which this project has no infrastructure
|
|
10
|
+
# for) just to exercise the non-sqlite path.
|
|
11
|
+
module Db
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# The web and worker processes each hold their own pool of connections to the
|
|
15
|
+
# same sqlite file, so writes from one process can easily collide with reads
|
|
16
|
+
# or writes from another. WAL mode lets readers proceed without blocking on a
|
|
17
|
+
# concurrent writer, and a longer busy_timeout gives a blocked writer more
|
|
18
|
+
# room to wait its turn instead of immediately raising SQLITE_BUSY.
|
|
19
|
+
#
|
|
20
|
+
# If you change this, run `bundle exec rake db:check_concurrency` to verify
|
|
21
|
+
# SQLITE_BUSY is still avoided under concurrent test-run writes and status
|
|
22
|
+
# polling (see lib/inferno/utils/db_concurrency_check.rb for why that's a
|
|
23
|
+
# rake task and not a spec).
|
|
24
|
+
def configure_sqlite_pragmas!(config)
|
|
25
|
+
return config unless config['adapter'] == 'sqlite'
|
|
26
|
+
|
|
27
|
+
connect_sqls = ["PRAGMA busy_timeout = #{ENV.fetch('DB_BUSY_TIMEOUT_MS', '15000')}"]
|
|
28
|
+
connect_sqls << 'PRAGMA journal_mode = WAL' unless config['database'] == ':memory:'
|
|
29
|
+
config['connect_sqls'] = connect_sqls
|
|
30
|
+
config
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def log_sqlite_journal_mode(config, connection, logger)
|
|
34
|
+
return unless config['adapter'] == 'sqlite'
|
|
35
|
+
|
|
36
|
+
actual_journal_mode = connection.fetch('PRAGMA journal_mode').first[:journal_mode]
|
|
37
|
+
logger.info("sqlite journal_mode: #{actual_journal_mode}")
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
4
44
|
Inferno::Application.register_provider(:db) do
|
|
5
45
|
prepare do
|
|
6
46
|
target_container.start :logging
|
|
@@ -13,6 +53,9 @@ Inferno::Application.register_provider(:db) do
|
|
|
13
53
|
config_contents = ERB.new(File.read(config_path)).result
|
|
14
54
|
config = YAML.safe_load(config_contents)[ENV.fetch('APP_ENV', nil)]
|
|
15
55
|
.merge(logger: Inferno::Application['logger'])
|
|
56
|
+
|
|
57
|
+
Inferno::Config::Boot::Db.configure_sqlite_pragmas!(config)
|
|
58
|
+
|
|
16
59
|
connection_attempts_remaining = ENV.fetch('MAX_DB_CONNECTION_ATTEMPTS', '10').to_i
|
|
17
60
|
connection_retry_delay = ENV.fetch('DB_CONNECTION_RETRY_DELAY', '5').to_i
|
|
18
61
|
connection = nil
|
|
@@ -31,6 +74,8 @@ Inferno::Application.register_provider(:db) do
|
|
|
31
74
|
end
|
|
32
75
|
connection.sql_log_level = :debug
|
|
33
76
|
|
|
77
|
+
Inferno::Config::Boot::Db.log_sqlite_journal_mode(config, connection, Inferno::Application['logger'])
|
|
78
|
+
|
|
34
79
|
register('db.config', config)
|
|
35
80
|
register('db.connection', connection)
|
|
36
81
|
end
|
|
@@ -7,6 +7,10 @@ Inferno::Application.register_provider(:executor) do
|
|
|
7
7
|
|
|
8
8
|
Blueprinter.configure do |config|
|
|
9
9
|
config.generator = Oj
|
|
10
|
+
# Oj >= 3.17 no longer honors ActiveSupport's Time#to_json, which
|
|
11
|
+
# dropped sub-second precision from serialized timestamps. Clients
|
|
12
|
+
# order results by these strings, so keep millisecond precision.
|
|
13
|
+
config.datetime_format = '%FT%T.%L%:z'
|
|
10
14
|
end
|
|
11
15
|
|
|
12
16
|
target_container.start :suites
|
|
@@ -5,6 +5,10 @@ Inferno::Application.register_provider(:web) do |_app|
|
|
|
5
5
|
|
|
6
6
|
Blueprinter.configure do |config|
|
|
7
7
|
config.generator = Oj
|
|
8
|
+
# Oj >= 3.17 no longer honors ActiveSupport's Time#to_json, which
|
|
9
|
+
# dropped sub-second precision from serialized timestamps. Clients
|
|
10
|
+
# order results by these strings, so keep millisecond precision.
|
|
11
|
+
config.datetime_format = '%FT%T.%L%:z'
|
|
8
12
|
end
|
|
9
13
|
|
|
10
14
|
# Workers aren't connected to a web server, so they shouldn't be hosting
|
|
@@ -193,14 +193,16 @@ module Inferno
|
|
|
193
193
|
inputs[input.name.to_sym] = Entities::Input.new(**input.to_hash)
|
|
194
194
|
end
|
|
195
195
|
|
|
196
|
+
# children_available_inputs walks the entire subtree, so it is computed once here
|
|
197
|
+
# rather than once per input. merge_with_child only mutates its receiver, so the
|
|
198
|
+
# child definitions are safe to reuse for the merge below.
|
|
199
|
+
child_inputs = children_available_inputs(selected_suite_options)
|
|
200
|
+
|
|
196
201
|
available_inputs.each do |input, current_definition|
|
|
197
|
-
|
|
198
|
-
current_definition.merge_with_child(child_definition)
|
|
202
|
+
current_definition.merge_with_child(child_inputs[input])
|
|
199
203
|
end
|
|
200
204
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
order_available_inputs(available_inputs)
|
|
205
|
+
order_available_inputs(child_inputs.merge(available_inputs))
|
|
204
206
|
end
|
|
205
207
|
end
|
|
206
208
|
end
|
|
@@ -94,7 +94,16 @@ module Inferno
|
|
|
94
94
|
update_result(waiting_result, result_message)
|
|
95
95
|
persist_request(request, test_run, waiting_result, test)
|
|
96
96
|
|
|
97
|
-
Jobs.perform(
|
|
97
|
+
Jobs.perform(
|
|
98
|
+
Jobs::ResumeTestRun,
|
|
99
|
+
test_run.id,
|
|
100
|
+
tags: [
|
|
101
|
+
'source:route',
|
|
102
|
+
"session:#{test_run.test_session_id}",
|
|
103
|
+
"run:#{test_run.test_suite_id || test_run.test_group_id || test_run.test_id}",
|
|
104
|
+
"test:#{test.id}"
|
|
105
|
+
]
|
|
106
|
+
)
|
|
98
107
|
|
|
99
108
|
res.redirect_to redirect_route(test_run, test)
|
|
100
109
|
end
|
|
@@ -254,6 +254,8 @@ module Inferno
|
|
|
254
254
|
def resume
|
|
255
255
|
req.env['inferno.resume_test_run'] = true
|
|
256
256
|
req.env['inferno.test_run_id'] = test_run.id
|
|
257
|
+
req.env['inferno.run_identifier'] = test_run.test_suite_id || test_run.test_group_id || test_run.test_id
|
|
258
|
+
req.env['inferno.waiting_test_id'] = test.id
|
|
257
259
|
end
|
|
258
260
|
|
|
259
261
|
# @private
|
|
@@ -318,7 +320,16 @@ module Inferno
|
|
|
318
320
|
test_run_id = env['inferno.test_run_id']
|
|
319
321
|
Inferno::Repositories::TestRuns.new.mark_as_no_longer_waiting(test_run_id)
|
|
320
322
|
|
|
321
|
-
Inferno::Jobs.perform(
|
|
323
|
+
Inferno::Jobs.perform(
|
|
324
|
+
Jobs::ResumeTestRun,
|
|
325
|
+
test_run_id,
|
|
326
|
+
tags: [
|
|
327
|
+
'source:suite_endpoint',
|
|
328
|
+
"session:#{env['inferno.test_session_id']}",
|
|
329
|
+
"run:#{env['inferno.run_identifier']}",
|
|
330
|
+
"test:#{env['inferno.waiting_test_id']}"
|
|
331
|
+
]
|
|
332
|
+
)
|
|
322
333
|
end
|
|
323
334
|
rescue StandardError => e
|
|
324
335
|
logger.error(e.full_message)
|
data/lib/inferno/jobs.rb
CHANGED
|
@@ -6,9 +6,13 @@ require_relative 'jobs/invoke_validator_session'
|
|
|
6
6
|
|
|
7
7
|
module Inferno
|
|
8
8
|
module Jobs
|
|
9
|
-
def self.perform(job_klass, *params, force_synchronous: false)
|
|
9
|
+
def self.perform(job_klass, *params, force_synchronous: false, tags: nil)
|
|
10
|
+
tags = Array(tags).compact
|
|
11
|
+
|
|
10
12
|
if force_synchronous || (Application['async_jobs'] == false)
|
|
11
13
|
job_klass.new.perform(*params)
|
|
14
|
+
elsif tags.any?
|
|
15
|
+
job_klass.set(tags:).perform_async(*params)
|
|
12
16
|
else
|
|
13
17
|
job_klass.perform_async(*params)
|
|
14
18
|
end
|
data/lib/inferno/test_runner.rb
CHANGED
|
@@ -75,20 +75,27 @@ module Inferno
|
|
|
75
75
|
outputs = save_outputs(test_instance)
|
|
76
76
|
output_json_string = JSON.generate(outputs)
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
78
|
+
result_params = {
|
|
79
|
+
messages: test_instance.messages,
|
|
80
|
+
requests: test_instance.requests,
|
|
81
|
+
result:,
|
|
82
|
+
result_message: test_instance.result_message,
|
|
83
|
+
input_json: input_json_string,
|
|
84
|
+
output_json: output_json_string
|
|
85
|
+
}.merge(test.reference_hash)
|
|
86
|
+
|
|
87
|
+
test_result =
|
|
88
|
+
if result == 'wait'
|
|
89
|
+
# The waiting status and the wait result must become visible to
|
|
90
|
+
# readers atomically, so that pollers never see the run waiting
|
|
91
|
+
# without the waiting test's result being present.
|
|
92
|
+
Inferno::Application['db.connection'].transaction do
|
|
93
|
+
test_runs_repo.mark_as_waiting(test_run.id, test_instance.identifier, test_instance.wait_timeout)
|
|
94
|
+
persist_result(result_params)
|
|
95
|
+
end
|
|
96
|
+
else
|
|
97
|
+
persist_result(result_params)
|
|
98
|
+
end
|
|
92
99
|
|
|
93
100
|
# If running a single test, update its parents' results. If running a
|
|
94
101
|
# group or suite, #run_group handles updating the parents.
|
|
@@ -249,9 +256,20 @@ module Inferno
|
|
|
249
256
|
end
|
|
250
257
|
|
|
251
258
|
def persist_result(params)
|
|
252
|
-
result
|
|
253
|
-
|
|
254
|
-
|
|
259
|
+
# A single result can cascade into many separate inserts (messages,
|
|
260
|
+
# requests, headers, tags). Wrapping them in one transaction turns that
|
|
261
|
+
# into a single lock acquisition instead of one per insert, cutting
|
|
262
|
+
# down on sqlite write-lock contention with concurrent readers/writers.
|
|
263
|
+
#
|
|
264
|
+
# If you change this, run `bundle exec rake db:check_concurrency` to verify
|
|
265
|
+
# SQLITE_BUSY is still avoided under concurrent test-run writes and status
|
|
266
|
+
# polling (see lib/inferno/utils/db_concurrency_check.rb for why that's a
|
|
267
|
+
# rake task and not a spec).
|
|
268
|
+
result = Inferno::Application['db.connection'].transaction do
|
|
269
|
+
results_repo.create(
|
|
270
|
+
params.merge(test_run_id: test_run.id, test_session_id: test_session.id)
|
|
271
|
+
)
|
|
272
|
+
end
|
|
255
273
|
|
|
256
274
|
run_results[result.runnable.id] = result
|
|
257
275
|
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
require 'sequel'
|
|
2
|
+
require 'sqlite3'
|
|
3
|
+
require 'tmpdir'
|
|
4
|
+
require 'securerandom'
|
|
5
|
+
|
|
6
|
+
module Inferno
|
|
7
|
+
module Utils
|
|
8
|
+
# Verifies that the sqlite WAL/busy_timeout/transaction-wrapping in
|
|
9
|
+
# lib/inferno/config/boot/db.rb and TestRunner#persist_result actually
|
|
10
|
+
# prevent SQLITE_BUSY under concurrent test-run writes and status-polling
|
|
11
|
+
# reads against a shared sqlite file.
|
|
12
|
+
#
|
|
13
|
+
# This is deliberately NOT an rspec example. It's a real-time concurrency
|
|
14
|
+
# measurement, and a hardcoded load level that reliably reproduces (or
|
|
15
|
+
# avoids) SQLITE_BUSY on one machine does not transfer to others: a load
|
|
16
|
+
# level tuned to fail reliably without the fix and pass reliably with it,
|
|
17
|
+
# verified repeatedly on one machine, failed consistently in GitHub
|
|
18
|
+
# Actions and on another local machine. Rather than assert against a
|
|
19
|
+
# fixed load level, this calibrates to whatever machine it runs on: it
|
|
20
|
+
# escalates the write volume until it can demonstrate the *unprotected*
|
|
21
|
+
# configuration actually hits SQLITE_BUSY, then checks whether the
|
|
22
|
+
# *protected* configuration avoids it under that same, machine-calibrated
|
|
23
|
+
# load.
|
|
24
|
+
#
|
|
25
|
+
# Run manually with `bundle exec rake db:check_concurrency` after changing
|
|
26
|
+
# the persistence/locking behavior in the files above. It isn't part of
|
|
27
|
+
# the default test suite because it's a real-time measurement, not a
|
|
28
|
+
# deterministic assertion, and shouldn't gate CI.
|
|
29
|
+
module DbConcurrencyCheck
|
|
30
|
+
WRITER_THREADS = 5
|
|
31
|
+
READER_THREADS = 3
|
|
32
|
+
READER_POLL_INTERVAL = 0.2
|
|
33
|
+
MAX_CONNECTIONS = 15
|
|
34
|
+
MAX_CASCADES_PER_WRITER = 5000
|
|
35
|
+
|
|
36
|
+
module_function
|
|
37
|
+
|
|
38
|
+
def run
|
|
39
|
+
cascades = 10
|
|
40
|
+
baseline_errors = []
|
|
41
|
+
|
|
42
|
+
loop do
|
|
43
|
+
baseline_errors = measure(protected: false, cascades_per_writer: cascades)
|
|
44
|
+
break if baseline_errors.any? || cascades > MAX_CASCADES_PER_WRITER
|
|
45
|
+
|
|
46
|
+
cascades *= 2
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
if baseline_errors.empty?
|
|
50
|
+
puts "INCONCLUSIVE: could not reproduce SQLITE_BUSY even at #{cascades} cascades/writer on this machine."
|
|
51
|
+
return true
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
puts "Reproduced #{baseline_errors.size} SQLITE_BUSY error(s) without WAL/busy_timeout/transaction-" \
|
|
55
|
+
"wrapping at #{cascades} cascades/writer (#{WRITER_THREADS} writers, #{READER_THREADS} readers " \
|
|
56
|
+
"polling every #{READER_POLL_INTERVAL}s)."
|
|
57
|
+
|
|
58
|
+
protected_errors = measure(protected: true, cascades_per_writer: cascades)
|
|
59
|
+
|
|
60
|
+
if protected_errors.empty?
|
|
61
|
+
puts 'PASS: the current fix avoided SQLITE_BUSY under the same load that reproduced it above.'
|
|
62
|
+
true
|
|
63
|
+
else
|
|
64
|
+
puts "FAIL: the current fix still hit #{protected_errors.size} SQLITE_BUSY error(s) under the same load:"
|
|
65
|
+
puts protected_errors.first.message
|
|
66
|
+
false
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def measure(protected:, cascades_per_writer:)
|
|
71
|
+
Dir.mktmpdir do |dir|
|
|
72
|
+
db = connect(File.join(dir, 'contention.db'), protected:)
|
|
73
|
+
result_class, request_class, header_class = build_schema_and_models(db)
|
|
74
|
+
errors = []
|
|
75
|
+
mutex = Mutex.new
|
|
76
|
+
stop_reading = false
|
|
77
|
+
|
|
78
|
+
writers = Array.new(WRITER_THREADS) do
|
|
79
|
+
Thread.new do
|
|
80
|
+
cascades_per_writer.times do
|
|
81
|
+
persist_cascade(db, result_class, request_class, header_class, protected:)
|
|
82
|
+
rescue StandardError => e
|
|
83
|
+
mutex.synchronize { errors << e }
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
readers = Array.new(READER_THREADS) do
|
|
89
|
+
Thread.new { poll_until(-> { stop_reading }, result_class, request_class, errors, mutex) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
writers.each(&:join)
|
|
93
|
+
stop_reading = true
|
|
94
|
+
readers.each(&:join)
|
|
95
|
+
db.disconnect
|
|
96
|
+
|
|
97
|
+
errors.select { |e| e.is_a?(Sequel::DatabaseError) && e.message =~ /database is locked|SQLITE_BUSY/i }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def connect(db_path, protected:)
|
|
102
|
+
connect_sqls =
|
|
103
|
+
if protected
|
|
104
|
+
["PRAGMA busy_timeout = #{ENV.fetch('DB_BUSY_TIMEOUT_MS', '15000')}", 'PRAGMA journal_mode = WAL']
|
|
105
|
+
else
|
|
106
|
+
['PRAGMA busy_timeout = 0']
|
|
107
|
+
end
|
|
108
|
+
Sequel.connect(adapter: 'sqlite', database: db_path, max_connections: MAX_CONNECTIONS, connect_sqls:)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def persist_cascade(db, result_class, request_class, header_class, protected:)
|
|
112
|
+
if protected
|
|
113
|
+
db.transaction { create_cascade(result_class, request_class, header_class) }
|
|
114
|
+
else
|
|
115
|
+
create_cascade(result_class, request_class, header_class)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def poll_until(stop, result_class, request_class, errors, mutex)
|
|
120
|
+
until stop.call
|
|
121
|
+
result_class.order(Sequel.desc(:id)).limit(1).all
|
|
122
|
+
request_class.order(Sequel.desc(:id)).limit(1).all
|
|
123
|
+
sleep(READER_POLL_INTERVAL)
|
|
124
|
+
end
|
|
125
|
+
rescue StandardError => e
|
|
126
|
+
mutex.synchronize { errors << e }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def build_schema_and_models(db)
|
|
130
|
+
db.create_table(:results) do
|
|
131
|
+
primary_key :id
|
|
132
|
+
column :test_id, String
|
|
133
|
+
column :result, String
|
|
134
|
+
end
|
|
135
|
+
db.create_table(:requests) do
|
|
136
|
+
primary_key :id
|
|
137
|
+
foreign_key :result_id, :results
|
|
138
|
+
column :verb, String
|
|
139
|
+
end
|
|
140
|
+
db.create_table(:headers) do
|
|
141
|
+
primary_key :id
|
|
142
|
+
foreign_key :request_id, :requests
|
|
143
|
+
column :name, String
|
|
144
|
+
column :value, String
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
[
|
|
148
|
+
Class.new(Sequel::Model(db[:results])),
|
|
149
|
+
Class.new(Sequel::Model(db[:requests])),
|
|
150
|
+
Class.new(Sequel::Model(db[:headers]))
|
|
151
|
+
]
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def create_cascade(result_class, request_class, header_class)
|
|
155
|
+
result = result_class.create(test_id: SecureRandom.uuid, result: 'pass')
|
|
156
|
+
3.times do
|
|
157
|
+
request = request_class.create(result_id: result.id, verb: 'GET')
|
|
158
|
+
2.times do |i|
|
|
159
|
+
header_class.create(request_id: request.id, name: "Header-#{i}", value: SecureRandom.hex(8))
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
result
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
data/lib/inferno/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: inferno_core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.4.
|
|
4
|
+
version: 1.4.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Stephen MacVicar
|
|
@@ -10,7 +10,7 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: bin
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2026-07-
|
|
13
|
+
date: 2026-07-27 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: activesupport
|
|
@@ -756,6 +756,7 @@ files:
|
|
|
756
756
|
- lib/inferno/route_storage.rb
|
|
757
757
|
- lib/inferno/spec_support.rb
|
|
758
758
|
- lib/inferno/test_runner.rb
|
|
759
|
+
- lib/inferno/utils/db_concurrency_check.rb
|
|
759
760
|
- lib/inferno/utils/execution_script_runner.rb
|
|
760
761
|
- lib/inferno/utils/ig_downloader.rb
|
|
761
762
|
- lib/inferno/utils/markdown_formatter.rb
|