polyrun 2.2.4 → 2.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/README.md +4 -1
- data/docs/SETUP_PROFILE.md +1 -0
- data/lib/polyrun/cli/database_commands.rb +21 -7
- data/lib/polyrun/cli/default_run.rb +28 -6
- data/lib/polyrun/cli/help.rb +4 -2
- data/lib/polyrun/cli/helpers.rb +41 -6
- data/lib/polyrun/cli/plan_command.rb +10 -29
- data/lib/polyrun/cli/run_queue_command.rb +10 -1
- data/lib/polyrun/cli/run_shards_command.rb +2 -8
- data/lib/polyrun/cli/run_shards_parallel_children.rb +52 -39
- data/lib/polyrun/cli/run_shards_parallel_wait.rb +2 -2
- data/lib/polyrun/cli/run_shards_plan_boot_phases.rb +3 -0
- data/lib/polyrun/cli/run_shards_planning.rb +16 -29
- data/lib/polyrun/cli/run_shards_run.rb +1 -1
- data/lib/polyrun/cli/run_shards_worker_interrupt.rb +16 -14
- data/lib/polyrun/cli/start_bootstrap.rb +51 -22
- data/lib/polyrun/cli.rb +8 -1
- data/lib/polyrun/config.rb +9 -4
- data/lib/polyrun/coverage/example_diff.rb +6 -7
- data/lib/polyrun/coverage/merge/formatters_html.rb +20 -2
- data/lib/polyrun/coverage/merge.rb +9 -3
- data/lib/polyrun/coverage/merge_fragment_meta.rb +6 -1
- data/lib/polyrun/coverage/rails.rb +1 -1
- data/lib/polyrun/database/clone_shards.rb +28 -5
- data/lib/polyrun/database/provision.rb +15 -9
- data/lib/polyrun/database/url_builder/connection/infer.rb +4 -0
- data/lib/polyrun/database/url_builder.rb +31 -3
- data/lib/polyrun/partition/paths.rb +23 -4
- data/lib/polyrun/partition/plan.rb +6 -24
- data/lib/polyrun/partition/suite.rb +116 -0
- data/lib/polyrun/partition/timing_diagnostics.rb +2 -3
- data/lib/polyrun/queue/file_store.rb +133 -30
- data/lib/polyrun/queue/file_store_intent.rb +188 -0
- data/lib/polyrun/queue/worker_loop.rb +41 -7
- data/lib/polyrun/spec_quality/config.rb +1 -1
- data/lib/polyrun/spec_quality/profile.rb +0 -1
- data/lib/polyrun/spec_quality/report.rb +16 -7
- data/lib/polyrun/templates/rails_prepare.polyrun.yml +3 -1
- data/lib/polyrun/version.rb +1 -1
- data/lib/polyrun/worker_output.rb +30 -13
- data/lib/polyrun.rb +1 -0
- metadata +3 -1
|
@@ -19,33 +19,38 @@ module Polyrun
|
|
|
19
19
|
# Best-effort worker teardown then exit. Does not return.
|
|
20
20
|
def run_shards_shutdown_on_signal!(pids, code)
|
|
21
21
|
run_shards_log_interrupt_workers(pids, nil)
|
|
22
|
-
Polyrun::WorkerOutput.shutdown_all!
|
|
23
22
|
run_shards_terminate_children!(pids)
|
|
24
23
|
exit(code)
|
|
25
24
|
rescue Interrupt
|
|
26
25
|
run_shards_signal_workers_kill(pids)
|
|
27
26
|
run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
|
|
27
|
+
Polyrun::WorkerOutput.shutdown_all!
|
|
28
28
|
exit(code)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
#
|
|
31
|
+
# Signal workers first so pipe readers see EOF; join forwarders only after children exit.
|
|
32
32
|
def run_shards_terminate_children!(pids)
|
|
33
|
-
Polyrun::WorkerOutput.shutdown_all!
|
|
34
33
|
run_shards_signal_workers_term(pids)
|
|
35
34
|
run_shards_reap_worker_pids_interruptible(pids.map { |h| h[:pid] })
|
|
35
|
+
Polyrun::WorkerOutput.shutdown_all!
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def run_shards_signal_workers_term(pids)
|
|
39
|
-
pids.each
|
|
40
|
-
Process.kill(:TERM, h[:pid])
|
|
41
|
-
rescue Errno::ESRCH
|
|
42
|
-
end
|
|
39
|
+
pids.each { |h| run_shards_kill_process_group!(:TERM, h[:pid]) }
|
|
43
40
|
end
|
|
44
41
|
|
|
45
42
|
def run_shards_signal_workers_kill(pids)
|
|
46
|
-
pids.each
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
pids.each { |h| run_shards_kill_process_group!(:KILL, h[:pid]) }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Negative PID signals the process group (workers spawn with +pgroup: true+).
|
|
47
|
+
def run_shards_kill_process_group!(signal, pid)
|
|
48
|
+
Process.kill(signal, -Integer(pid))
|
|
49
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
50
|
+
begin
|
|
51
|
+
Process.kill(signal, Integer(pid))
|
|
52
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
53
|
+
nil
|
|
49
54
|
end
|
|
50
55
|
end
|
|
51
56
|
|
|
@@ -71,10 +76,7 @@ module Polyrun
|
|
|
71
76
|
force_note = true
|
|
72
77
|
Polyrun::Log.orchestration_warn "polyrun run-shards: repeated SIGINT during worker cleanup — SIGKILL to #{pending.size} process(es)"
|
|
73
78
|
end
|
|
74
|
-
pending.each
|
|
75
|
-
Process.kill(:KILL, pid)
|
|
76
|
-
rescue Errno::ESRCH
|
|
77
|
-
end
|
|
79
|
+
pending.each { |pid| run_shards_kill_process_group!(:KILL, pid) }
|
|
78
80
|
end
|
|
79
81
|
end
|
|
80
82
|
end
|
|
@@ -12,24 +12,30 @@ module Polyrun
|
|
|
12
12
|
return code if code != 0
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
15
|
+
return 0 unless start_databases_provision_wanted?(cfg) && !truthy_env?("POLYRUN_START_SKIP_DATABASES")
|
|
16
|
+
|
|
17
|
+
dh = cfg.databases
|
|
18
|
+
adapter = Polyrun::Database::UrlBuilder::ConnectionInfer.infer_adapter_name(dh)
|
|
19
|
+
unless adapter == "postgresql"
|
|
20
|
+
return start_non_postgresql_databases_outcome(cfg, adapter)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
workers = parse_workers_from_start_argv(argv)
|
|
24
|
+
Polyrun::Log.warn "polyrun start: provisioning test DBs (template + shards 0..#{workers - 1})" if @verbose
|
|
25
|
+
begin
|
|
26
|
+
Polyrun::Database::CloneShards.provision!(
|
|
27
|
+
dh,
|
|
28
|
+
workers: workers,
|
|
29
|
+
rails_root: Dir.pwd,
|
|
30
|
+
migrate: true,
|
|
31
|
+
replace: true,
|
|
32
|
+
force_drop: truthy_env?("POLYRUN_PG_DROP_FORCE"),
|
|
33
|
+
dry_run: false,
|
|
34
|
+
silent: !@verbose
|
|
35
|
+
)
|
|
36
|
+
rescue Polyrun::Error => e
|
|
37
|
+
Polyrun::Log.warn "polyrun start: #{e.message}"
|
|
38
|
+
return 1
|
|
33
39
|
end
|
|
34
40
|
0
|
|
35
41
|
end
|
|
@@ -53,7 +59,7 @@ module Polyrun
|
|
|
53
59
|
false
|
|
54
60
|
end
|
|
55
61
|
|
|
56
|
-
def
|
|
62
|
+
def start_databases_provision_wanted?(cfg)
|
|
57
63
|
st = cfg.start_config
|
|
58
64
|
dh = cfg.databases
|
|
59
65
|
return false unless dh.is_a?(Hash)
|
|
@@ -61,14 +67,33 @@ module Polyrun
|
|
|
61
67
|
template = (dh["template_db"] || dh[:template_db]).to_s
|
|
62
68
|
return false if template.empty?
|
|
63
69
|
|
|
64
|
-
if st["databases"] == true || st[:databases] == true
|
|
65
|
-
return true
|
|
66
|
-
end
|
|
67
70
|
return false if st["databases"] == false || st[:databases] == false
|
|
68
71
|
|
|
69
72
|
true
|
|
70
73
|
end
|
|
71
74
|
|
|
75
|
+
def start_databases_explicitly_enabled?(cfg)
|
|
76
|
+
st = cfg.start_config
|
|
77
|
+
st["databases"] == true || st[:databases] == true
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def start_non_postgresql_databases_outcome(cfg, adapter)
|
|
81
|
+
if start_databases_explicitly_enabled?(cfg)
|
|
82
|
+
Polyrun::Log.warn(
|
|
83
|
+
"polyrun start: start.databases is true but databases adapter is #{adapter}; " \
|
|
84
|
+
"CloneShards auto provision is PostgreSQL-only. Set start.databases: false and provision " \
|
|
85
|
+
"shards yourself (for example in prepare), or use a postgresql: connection block."
|
|
86
|
+
)
|
|
87
|
+
return 1
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
Polyrun::Log.warn(
|
|
91
|
+
"polyrun start: skipping database provision (adapter=#{adapter}; CloneShards is PostgreSQL-only). " \
|
|
92
|
+
"Provision shards in prepare, or set start.databases: false to silence this message."
|
|
93
|
+
)
|
|
94
|
+
0
|
|
95
|
+
end
|
|
96
|
+
|
|
72
97
|
def parse_workers_from_start_argv(argv)
|
|
73
98
|
sep = argv.index("--")
|
|
74
99
|
head = sep ? argv[0...sep] : argv
|
|
@@ -79,6 +104,10 @@ module Polyrun
|
|
|
79
104
|
w = Integer(head[i + 1], exception: false)
|
|
80
105
|
workers = w if w && w >= 1
|
|
81
106
|
i += 2
|
|
107
|
+
elsif head[i].start_with?("--workers=")
|
|
108
|
+
w = Integer(head[i].split("=", 2).last, exception: false)
|
|
109
|
+
workers = w if w && w >= 1
|
|
110
|
+
i += 1
|
|
82
111
|
else
|
|
83
112
|
i += 1
|
|
84
113
|
end
|
data/lib/polyrun/cli.rb
CHANGED
|
@@ -35,7 +35,7 @@ module Polyrun
|
|
|
35
35
|
DISPATCH_SUBCOMMAND_NAMES = %w[
|
|
36
36
|
plan prepare merge-coverage merge-failures merge-spec-quality report-coverage report-junit report-timing report-spec-quality report-benchmark bench
|
|
37
37
|
env config merge-timing db:setup-template db:setup-shard db:clone-shards
|
|
38
|
-
run-shards parallel-rspec start build-paths init queue run-queue quick hook
|
|
38
|
+
run-shards parallel-rspec parallel-minitest parallel-quick start build-paths init queue run-queue quick hook
|
|
39
39
|
].freeze
|
|
40
40
|
|
|
41
41
|
# First argv token that is a normal subcommand (not a path); if argv[0] is not here but looks like paths, run implicit parallel.
|
|
@@ -108,6 +108,9 @@ module Polyrun
|
|
|
108
108
|
)
|
|
109
109
|
|
|
110
110
|
dispatch_cli_command(command, argv, config_path)
|
|
111
|
+
rescue Polyrun::Error => e
|
|
112
|
+
Polyrun::Log.warn "polyrun: #{e.message}"
|
|
113
|
+
2
|
|
111
114
|
end
|
|
112
115
|
|
|
113
116
|
private
|
|
@@ -187,6 +190,10 @@ module Polyrun
|
|
|
187
190
|
cmd_run_shards(argv, config_path)
|
|
188
191
|
when "parallel-rspec"
|
|
189
192
|
cmd_parallel_rspec(argv, config_path)
|
|
193
|
+
when "parallel-minitest"
|
|
194
|
+
cmd_parallel_minitest(argv, config_path)
|
|
195
|
+
when "parallel-quick"
|
|
196
|
+
cmd_parallel_quick(argv, config_path)
|
|
190
197
|
when "start"
|
|
191
198
|
cmd_start(argv, config_path)
|
|
192
199
|
when "build-paths"
|
data/lib/polyrun/config.rb
CHANGED
|
@@ -12,14 +12,19 @@ module Polyrun
|
|
|
12
12
|
attr_reader :path, :raw
|
|
13
13
|
|
|
14
14
|
def self.load(path: nil)
|
|
15
|
-
|
|
15
|
+
explicit = path && !path.to_s.strip.empty?
|
|
16
|
+
resolved = resolve_path(path)
|
|
17
|
+
if explicit && !(resolved && File.file?(resolved))
|
|
18
|
+
raise Polyrun::Error, "polyrun config not found: #{path}"
|
|
19
|
+
end
|
|
20
|
+
|
|
16
21
|
raw =
|
|
17
|
-
if
|
|
18
|
-
YAML.safe_load_file(
|
|
22
|
+
if resolved && File.file?(resolved)
|
|
23
|
+
YAML.safe_load_file(resolved, permitted_classes: [Symbol], aliases: true) || {}
|
|
19
24
|
else
|
|
20
25
|
{}
|
|
21
26
|
end
|
|
22
|
-
new(path:
|
|
27
|
+
new(path: resolved, raw: raw)
|
|
23
28
|
end
|
|
24
29
|
|
|
25
30
|
def self.resolve_path(explicit)
|
|
@@ -131,14 +131,13 @@ module Polyrun
|
|
|
131
131
|
end
|
|
132
132
|
|
|
133
133
|
def collect_diff_paths(before_blob, after_blob, filter: nil)
|
|
134
|
-
keys =
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
return keys unless filter
|
|
134
|
+
keys = {}
|
|
135
|
+
before_blob.each_key { |path| keys[path.to_s] = true }
|
|
136
|
+
after_blob.each_key { |path| keys[path.to_s] = true }
|
|
137
|
+
list = keys.keys
|
|
138
|
+
return list unless filter
|
|
140
139
|
|
|
141
|
-
|
|
140
|
+
list.select { |path| filter.include_path?(path) }
|
|
142
141
|
end
|
|
143
142
|
|
|
144
143
|
def accumulate_line_delta(before_entry, after_lines)
|
|
@@ -23,7 +23,7 @@ module Polyrun
|
|
|
23
23
|
)
|
|
24
24
|
file_list_html = render_html_partial("file_list", file_rows_html: files.map { |file| html_file_list_row(file) }.join("\n"))
|
|
25
25
|
file_sections_html = files.map { |file| render_html_partial("file_section", file: file) }.join("\n")
|
|
26
|
-
|
|
26
|
+
html_root_template_erb.result_with_hash(
|
|
27
27
|
title: CGI.escapeHTML(title.to_s),
|
|
28
28
|
generated_label: html_generated_label(generated_at),
|
|
29
29
|
overview_html: overview_html,
|
|
@@ -56,9 +56,27 @@ module Polyrun
|
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
def render_html_partial(name, locals = {})
|
|
59
|
-
|
|
59
|
+
html_partial_erb(name).result_with_hash(locals)
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
# rubocop:disable ThreadSafety/ClassInstanceVariable -- template cache keyed by partial name; process-local
|
|
63
|
+
def html_partial_erb(name)
|
|
64
|
+
@html_partial_erbs ||= {}
|
|
65
|
+
@html_partial_erbs[name] ||= ERB.new(
|
|
66
|
+
File.read(html_partial_path(name), encoding: Encoding::UTF_8),
|
|
67
|
+
trim_mode: "-"
|
|
68
|
+
)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def html_root_template_erb
|
|
72
|
+
@html_root_template_erb ||= ERB.new(
|
|
73
|
+
File.read(html_template_path, encoding: Encoding::UTF_8),
|
|
74
|
+
trim_mode: "-"
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
# rubocop:enable ThreadSafety/ClassInstanceVariable
|
|
78
|
+
private_class_method :html_partial_erb, :html_root_template_erb
|
|
79
|
+
|
|
62
80
|
def html_file_payload(path, file, root)
|
|
63
81
|
line_arr = line_array_from_file_entry(file) || []
|
|
64
82
|
source_lines = html_source_lines(path, line_arr.length)
|
|
@@ -26,10 +26,16 @@ module Polyrun
|
|
|
26
26
|
def merge_fragments(paths)
|
|
27
27
|
return {blob: {}, meta: {}, groups: nil} if paths.empty?
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
# Parse one fragment at a time: keep only meta + blob so peak RSS drops after each fold step.
|
|
30
|
+
metas = []
|
|
31
|
+
blobs = []
|
|
32
|
+
paths.each do |path|
|
|
33
|
+
doc = JSON.parse(File.read(path))
|
|
34
|
+
metas << extract_doc_meta(doc)
|
|
35
|
+
blobs << extract_coverage_blob(doc)
|
|
36
|
+
end
|
|
31
37
|
merged_blob = merge_blob_tree(blobs)
|
|
32
|
-
merged_meta =
|
|
38
|
+
merged_meta = merge_fragment_metas_from_metas(metas)
|
|
33
39
|
merged_blob = apply_track_files_once_after_merge(merged_blob, merged_meta)
|
|
34
40
|
groups_payload = recompute_groups_from_meta(merged_blob, merged_meta)
|
|
35
41
|
{blob: merged_blob, meta: merged_meta, groups: groups_payload}
|
|
@@ -4,7 +4,12 @@ module Polyrun
|
|
|
4
4
|
module_function
|
|
5
5
|
|
|
6
6
|
def merge_fragment_metas(docs)
|
|
7
|
-
|
|
7
|
+
merge_fragment_metas_from_metas(docs.map { |d| extract_doc_meta(d) })
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def merge_fragment_metas_from_metas(metas)
|
|
11
|
+
return {} if metas.empty?
|
|
12
|
+
|
|
8
13
|
base = metas.first.dup
|
|
9
14
|
roots = metas.map { |m| m["polyrun_coverage_root"] }.compact
|
|
10
15
|
grs = metas.map { |m| m["polyrun_coverage_groups"] }.compact
|
|
@@ -9,6 +9,7 @@ module Polyrun
|
|
|
9
9
|
# See +provision!+ on the singleton class for options.
|
|
10
10
|
def provision!(databases_hash, workers:, rails_root:, migrate: true, replace: true, force_drop: false, dry_run: false, silent: true)
|
|
11
11
|
dh = databases_hash.is_a?(Hash) ? databases_hash : {}
|
|
12
|
+
ensure_postgresql!(dh)
|
|
12
13
|
workers = Integer(workers)
|
|
13
14
|
raise Polyrun::Error, "workers must be >= 1" if workers < 1
|
|
14
15
|
|
|
@@ -19,6 +20,16 @@ module Polyrun
|
|
|
19
20
|
true
|
|
20
21
|
end
|
|
21
22
|
|
|
23
|
+
def ensure_postgresql!(dh)
|
|
24
|
+
return if UrlBuilder::ConnectionInfer.postgresql?(dh)
|
|
25
|
+
|
|
26
|
+
adapter = UrlBuilder::ConnectionInfer.infer_adapter_name(dh)
|
|
27
|
+
raise Polyrun::Error,
|
|
28
|
+
"CloneShards is PostgreSQL-only (databases adapter is #{adapter}). " \
|
|
29
|
+
"Use polyrun env for URLs and provision shards yourself."
|
|
30
|
+
end
|
|
31
|
+
private_class_method :ensure_postgresql!
|
|
32
|
+
|
|
22
33
|
def migrate_canonical_databases!(dh, rails_root, dry_run, silent)
|
|
23
34
|
pt = (dh["template_db"] || dh[:template_db]).to_s
|
|
24
35
|
if pt.empty?
|
|
@@ -37,6 +48,7 @@ module Polyrun
|
|
|
37
48
|
private_class_method :migrate_canonical_databases!
|
|
38
49
|
|
|
39
50
|
def create_shards_from_plan!(dh, workers, replace, force_drop, dry_run)
|
|
51
|
+
conn = UrlBuilder::Connection.resolve_connection(dh)
|
|
40
52
|
if dry_run
|
|
41
53
|
workers.times do |shard_index|
|
|
42
54
|
plan = UrlBuilder.shard_database_plan(dh, shard_index: shard_index)
|
|
@@ -44,7 +56,7 @@ module Polyrun
|
|
|
44
56
|
raise Polyrun::Error, "CloneShards: empty shard plan for shard_index=#{shard_index}"
|
|
45
57
|
end
|
|
46
58
|
|
|
47
|
-
plan.each { |row| create_one_shard!(row, replace, force_drop, dry_run) }
|
|
59
|
+
plan.each { |row| create_one_shard!(row, replace, force_drop, dry_run, conn) }
|
|
48
60
|
end
|
|
49
61
|
return
|
|
50
62
|
end
|
|
@@ -56,7 +68,7 @@ module Polyrun
|
|
|
56
68
|
raise Polyrun::Error, "CloneShards: empty shard plan for shard_index=#{shard_index}"
|
|
57
69
|
end
|
|
58
70
|
|
|
59
|
-
plan.each { |row| create_one_shard!(row, replace, force_drop, dry_run) }
|
|
71
|
+
plan.each { |row| create_one_shard!(row, replace, force_drop, dry_run, conn) }
|
|
60
72
|
rescue => e
|
|
61
73
|
raise Polyrun::Error, "CloneShards shard_index=#{shard_index}: #{e.message}"
|
|
62
74
|
end
|
|
@@ -65,7 +77,7 @@ module Polyrun
|
|
|
65
77
|
end
|
|
66
78
|
private_class_method :create_shards_from_plan!
|
|
67
79
|
|
|
68
|
-
def create_one_shard!(row, replace, force_drop, dry_run)
|
|
80
|
+
def create_one_shard!(row, replace, force_drop, dry_run, conn)
|
|
69
81
|
new_db = row[:new_db].to_s
|
|
70
82
|
tmpl = row[:template_db].to_s
|
|
71
83
|
if dry_run
|
|
@@ -74,10 +86,21 @@ module Polyrun
|
|
|
74
86
|
return
|
|
75
87
|
end
|
|
76
88
|
|
|
77
|
-
|
|
78
|
-
Provision.
|
|
89
|
+
opts = provision_connection_options(conn)
|
|
90
|
+
Provision.drop_database_if_exists!(database: new_db, force: force_drop, **opts) if replace
|
|
91
|
+
Provision.create_database_from_template!(new_db: new_db, template_db: tmpl, **opts)
|
|
79
92
|
end
|
|
80
93
|
private_class_method :create_one_shard!
|
|
94
|
+
|
|
95
|
+
def provision_connection_options(conn)
|
|
96
|
+
{
|
|
97
|
+
host: conn[:host],
|
|
98
|
+
port: conn[:port],
|
|
99
|
+
username: conn[:user],
|
|
100
|
+
password: conn[:password]
|
|
101
|
+
}
|
|
102
|
+
end
|
|
103
|
+
private_class_method :provision_connection_options
|
|
81
104
|
end
|
|
82
105
|
end
|
|
83
106
|
end
|
|
@@ -14,7 +14,7 @@ module Polyrun
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
# +DROP DATABASE IF EXISTS name;+ — maintenance DB +postgres+ (or +maintenance_db+).
|
|
17
|
-
def drop_database_if_exists!(database:, host: nil, port: nil, username: nil, maintenance_db: "postgres", force: false)
|
|
17
|
+
def drop_database_if_exists!(database:, host: nil, port: nil, username: nil, password: nil, maintenance_db: "postgres", force: false)
|
|
18
18
|
host ||= ENV["PGHOST"] || "localhost"
|
|
19
19
|
port ||= ENV["PGPORT"] || "5432"
|
|
20
20
|
username ||= ENV["PGUSER"] || "postgres"
|
|
@@ -25,26 +25,32 @@ module Polyrun
|
|
|
25
25
|
else
|
|
26
26
|
"DROP DATABASE IF EXISTS #{quote_ident(database)};"
|
|
27
27
|
end
|
|
28
|
-
|
|
29
|
-
_out, err, st = Open3.capture3(*cmd)
|
|
30
|
-
raise Polyrun::Error, "drop database failed: #{err}" unless st.success?
|
|
31
|
-
|
|
32
|
-
true
|
|
28
|
+
run_psql!(sql, host: host, port: port, username: username, password: password, maintenance_db: maintenance_db, failure_prefix: "drop database failed")
|
|
33
29
|
end
|
|
34
30
|
|
|
35
31
|
# CREATE DATABASE new_db TEMPLATE template_db — connects to maintenance DB +postgres+.
|
|
36
|
-
def create_database_from_template!(new_db:, template_db:, host: nil, port: nil, username: nil, maintenance_db: "postgres")
|
|
32
|
+
def create_database_from_template!(new_db:, template_db:, host: nil, port: nil, username: nil, password: nil, maintenance_db: "postgres")
|
|
37
33
|
host ||= ENV["PGHOST"] || "localhost"
|
|
38
34
|
port ||= ENV["PGPORT"] || "5432"
|
|
39
35
|
username ||= ENV["PGUSER"] || "postgres"
|
|
40
36
|
|
|
41
37
|
sql = "CREATE DATABASE #{quote_ident(new_db)} TEMPLATE #{quote_ident(template_db)};"
|
|
38
|
+
run_psql!(sql, host: host, port: port, username: username, password: password, maintenance_db: maintenance_db, failure_prefix: "create database failed")
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def run_psql!(sql, host:, port:, username:, password:, maintenance_db:, failure_prefix:)
|
|
42
42
|
cmd = ["psql", "-U", username, "-h", host, "-p", port.to_s, "-d", maintenance_db, "-v", "ON_ERROR_STOP=1", "-c", sql]
|
|
43
|
-
_out, err, st =
|
|
44
|
-
|
|
43
|
+
_out, err, st =
|
|
44
|
+
if password.nil? || password.to_s.empty?
|
|
45
|
+
Open3.capture3(*cmd)
|
|
46
|
+
else
|
|
47
|
+
Open3.capture3(ENV.to_h.merge("PGPASSWORD" => password.to_s), *cmd)
|
|
48
|
+
end
|
|
49
|
+
raise Polyrun::Error, "#{failure_prefix}: #{err}" unless st.success?
|
|
45
50
|
|
|
46
51
|
true
|
|
47
52
|
end
|
|
53
|
+
private_class_method :run_psql!
|
|
48
54
|
|
|
49
55
|
# Runs +bin/rails db:prepare+ with merged ENV (+DATABASE_URL+ for primary, +CACHE_DATABASE_URL+, etc.).
|
|
50
56
|
# Multi-DB Rails apps must pass all template URLs in one invocation so each DB uses its own +migrations_paths+.
|
|
@@ -29,6 +29,10 @@ module Polyrun
|
|
|
29
29
|
"postgresql"
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
+
def postgresql?(databases_hash)
|
|
33
|
+
infer_adapter_name(databases_hash.is_a?(Hash) ? databases_hash : {}) == "postgresql"
|
|
34
|
+
end
|
|
35
|
+
|
|
32
36
|
def normalize_adapter_alias(name)
|
|
33
37
|
case name
|
|
34
38
|
when "postgres", "pg" then "postgresql"
|
|
@@ -48,7 +48,9 @@ module Polyrun
|
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
def template_prepare_env_shell_log(databases_hash)
|
|
51
|
-
template_prepare_env(databases_hash).sort.map
|
|
51
|
+
template_prepare_env(databases_hash).sort.map do |k, v|
|
|
52
|
+
"#{k}=#{Shellwords.escape(redact_url_userinfo_password(v.to_s))}"
|
|
53
|
+
end.join(" ")
|
|
52
54
|
end
|
|
53
55
|
|
|
54
56
|
def unique_template_migrate_urls(databases_hash)
|
|
@@ -111,6 +113,19 @@ module Polyrun
|
|
|
111
113
|
def url_for_shard(databases_hash, shard_index:, connection: nil)
|
|
112
114
|
dh = databases_hash.is_a?(Hash) ? databases_hash : {}
|
|
113
115
|
conn = Connection.resolve_connection(dh)
|
|
116
|
+
pattern = shard_pattern(dh, connection)
|
|
117
|
+
validate_shard_db_pattern!(pattern)
|
|
118
|
+
|
|
119
|
+
dbname = pattern.to_s.gsub("%{shard}", Integer(shard_index).to_s).gsub("%<shard>d", format("%d", Integer(shard_index)))
|
|
120
|
+
template_name = template_database_name_for(dh, connection: connection)
|
|
121
|
+
if !template_name.to_s.empty? && dbname == template_name.to_s
|
|
122
|
+
raise Polyrun::Error,
|
|
123
|
+
"databases.shard_db_pattern resolves to template_db #{template_name.inspect} for shard #{shard_index}"
|
|
124
|
+
end
|
|
125
|
+
Connection.build_database_url(dbname, conn)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def shard_pattern(dh, connection)
|
|
114
129
|
pattern =
|
|
115
130
|
if connection
|
|
116
131
|
conns = dh["connections"] || dh[:connections] || []
|
|
@@ -120,10 +135,23 @@ module Polyrun
|
|
|
120
135
|
dh["shard_db_pattern"] || dh[:shard_db_pattern]
|
|
121
136
|
end
|
|
122
137
|
pattern ||= "app_test_%{shard}"
|
|
138
|
+
pattern
|
|
139
|
+
end
|
|
140
|
+
private_class_method :shard_pattern
|
|
123
141
|
|
|
124
|
-
|
|
125
|
-
|
|
142
|
+
def validate_shard_db_pattern!(pattern)
|
|
143
|
+
p = pattern.to_s
|
|
144
|
+
return if p.include?("%{shard}") || p.include?("%<shard>d")
|
|
145
|
+
|
|
146
|
+
raise Polyrun::Error, "databases.shard_db_pattern must include %{shard} or %<shard>d (got #{p.inspect})"
|
|
147
|
+
end
|
|
148
|
+
private_class_method :validate_shard_db_pattern!
|
|
149
|
+
|
|
150
|
+
def redact_url_userinfo_password(url)
|
|
151
|
+
s = url.to_s
|
|
152
|
+
s.sub(%r{\A([a-z][a-z0-9+.-]*://[^:/?#]+:)[^@/?#]+(@)}i, '\1***\2')
|
|
126
153
|
end
|
|
154
|
+
private_class_method :redact_url_userinfo_password
|
|
127
155
|
|
|
128
156
|
def env_exports_for_databases(databases_hash, shard_index:)
|
|
129
157
|
dh = databases_hash.is_a?(Hash) ? databases_hash : {}
|
|
@@ -23,9 +23,10 @@ module Polyrun
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
# Infer parallel suite from explicit paths (+_spec.rb+ vs +_test.rb+ vs Polyrun quick-style +.rb+).
|
|
26
|
+
# Recognizes RSpec-style +path:line+ locators (strips a trailing numeric locator before classifying).
|
|
26
27
|
# Returns +:rspec+, +:minitest+, +:quick+, +:invalid+ (mixed spec and test), or +nil+ (empty).
|
|
27
28
|
def infer_suite_from_paths(paths)
|
|
28
|
-
paths = paths.map { |p|
|
|
29
|
+
paths = Array(paths).map { |p| suite_classify_path(p) }
|
|
29
30
|
return nil if paths.empty?
|
|
30
31
|
|
|
31
32
|
specs = paths.count { |p| File.basename(p).end_with?("_spec.rb") }
|
|
@@ -44,8 +45,10 @@ module Polyrun
|
|
|
44
45
|
# When +paths_file+ is set but missing, returns +{ error: "..." }+.
|
|
45
46
|
# Otherwise returns +{ items:, source: }+ (human-readable source label).
|
|
46
47
|
#
|
|
47
|
-
# +partition.suite+ (optional): +auto+ (default), +rspec+, +minitest+, +quick+ — used
|
|
48
|
-
# from globs
|
|
48
|
+
# +partition.suite+ (optional): +auto+ (default), +rspec+, +minitest+, +quick+ — used when resolving
|
|
49
|
+
# from globs. Legacy +spec/spec_paths.txt+ is used only for +auto+/+rspec+ (RSpec-oriented); an
|
|
50
|
+
# explicit +minitest+/+quick+ suite skips it and globs instead. Bare +polyrun+ suite selection is
|
|
51
|
+
# {Suite.resolve_default} (explicit suite → paths_file inference → {detect_auto_suite}).
|
|
49
52
|
def resolve_run_shard_items(paths_file: nil, cwd: Dir.pwd, partition: {})
|
|
50
53
|
if paths_file
|
|
51
54
|
abs = File.expand_path(paths_file.to_s, cwd)
|
|
@@ -53,7 +56,7 @@ module Polyrun
|
|
|
53
56
|
return {error: "paths file not found: #{abs}"}
|
|
54
57
|
end
|
|
55
58
|
{items: read_lines(abs), source: paths_file.to_s}
|
|
56
|
-
elsif
|
|
59
|
+
elsif use_legacy_spec_paths?(cwd, partition)
|
|
57
60
|
{items: read_lines(File.join(cwd, "spec", "spec_paths.txt")), source: "spec/spec_paths.txt"}
|
|
58
61
|
else
|
|
59
62
|
resolve_run_shard_items_glob(cwd: cwd, partition: partition)
|
|
@@ -104,6 +107,22 @@ module Polyrun
|
|
|
104
107
|
require_relative "../quick/runner"
|
|
105
108
|
Polyrun::Quick::Runner.parallel_default_paths(base)
|
|
106
109
|
end
|
|
110
|
+
|
|
111
|
+
# Strip trailing numeric locators (+path:line+ or +path:line:column+), then expand.
|
|
112
|
+
def suite_classify_path(path)
|
|
113
|
+
raw = path.to_s.sub(/(?::\d+)+\z/, "")
|
|
114
|
+
File.expand_path(raw)
|
|
115
|
+
end
|
|
116
|
+
private_class_method :suite_classify_path
|
|
117
|
+
|
|
118
|
+
def use_legacy_spec_paths?(cwd, partition)
|
|
119
|
+
return false unless File.file?(File.join(cwd, "spec", "spec_paths.txt"))
|
|
120
|
+
|
|
121
|
+
suite = (partition["suite"] || partition[:suite] || "auto").to_s.downcase
|
|
122
|
+
suite = "auto" if suite.empty?
|
|
123
|
+
%w[auto rspec].include?(suite)
|
|
124
|
+
end
|
|
125
|
+
private_class_method :use_legacy_spec_paths?
|
|
107
126
|
end
|
|
108
127
|
end
|
|
109
128
|
end
|
|
@@ -110,11 +110,9 @@ module Polyrun
|
|
|
110
110
|
end
|
|
111
111
|
|
|
112
112
|
def default_weight
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
1.0
|
|
116
|
-
else
|
|
117
|
-
vals.sum / vals.size
|
|
113
|
+
@default_weight ||= begin
|
|
114
|
+
vals = @costs&.values || []
|
|
115
|
+
vals.empty? ? 1.0 : vals.sum / vals.size
|
|
118
116
|
end
|
|
119
117
|
end
|
|
120
118
|
|
|
@@ -193,13 +191,7 @@ module Polyrun
|
|
|
193
191
|
|
|
194
192
|
out = {}
|
|
195
193
|
map.each do |k, v|
|
|
196
|
-
|
|
197
|
-
if @timing_granularity == :example
|
|
198
|
-
TimingKeys.normalize_locator(k.to_s, @root, :example)
|
|
199
|
-
else
|
|
200
|
-
File.expand_path(k.to_s, @root)
|
|
201
|
-
end
|
|
202
|
-
out[key] = Integer(v)
|
|
194
|
+
out[TimingKeys.normalize_locator(k.to_s, @root, @timing_granularity)] = Integer(v)
|
|
203
195
|
end
|
|
204
196
|
out
|
|
205
197
|
end
|
|
@@ -209,13 +201,7 @@ module Polyrun
|
|
|
209
201
|
|
|
210
202
|
c = {}
|
|
211
203
|
costs.each do |k, v|
|
|
212
|
-
|
|
213
|
-
if @timing_granularity == :example
|
|
214
|
-
TimingKeys.normalize_locator(k.to_s, @root, :example)
|
|
215
|
-
else
|
|
216
|
-
File.expand_path(k.to_s, @root)
|
|
217
|
-
end
|
|
218
|
-
c[key] = v.to_f
|
|
204
|
+
c[TimingKeys.normalize_locator(k.to_s, @root, @timing_granularity)] = v.to_f
|
|
219
205
|
end
|
|
220
206
|
c
|
|
221
207
|
end
|
|
@@ -243,11 +229,7 @@ module Polyrun
|
|
|
243
229
|
end
|
|
244
230
|
|
|
245
231
|
def cost_lookup_key(path)
|
|
246
|
-
|
|
247
|
-
TimingKeys.normalize_locator(path, @root, :example)
|
|
248
|
-
else
|
|
249
|
-
File.expand_path(path, @root)
|
|
250
|
-
end
|
|
232
|
+
TimingKeys.normalize_locator(path, @root, @timing_granularity)
|
|
251
233
|
end
|
|
252
234
|
|
|
253
235
|
def cost_shards
|