fun_ci 2.3.0-aarch64-linux → 2.5.0-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +143 -0
- data/README.md +20 -10
- data/lib/fun_ci/agent/events.rb +15 -7
- data/lib/fun_ci/agent/job_events.rb +12 -5
- data/lib/fun_ci/agent/onward.rb +21 -0
- data/lib/fun_ci/agent/output.rb +7 -0
- data/lib/fun_ci/agent/reports.rb +3 -0
- data/lib/fun_ci/agent/snapshots.rb +25 -7
- data/lib/fun_ci/agent/status_text.rb +9 -5
- data/lib/fun_ci/agent/wait_command.rb +37 -5
- data/lib/fun_ci/cli.rb +2 -0
- data/lib/fun_ci/cli_command_help.rb +51 -0
- data/lib/fun_ci/cli_help.rb +9 -9
- data/lib/fun_ci/console/board_data.rb +50 -10
- data/lib/fun_ci/console/branch_page.rb +9 -0
- data/lib/fun_ci/console/console_state.rb +22 -8
- data/lib/fun_ci/console/cursor.rb +22 -0
- data/lib/fun_ci/console/cursor_rows.rb +37 -0
- data/lib/fun_ci/console/job_message.rb +5 -4
- data/lib/fun_ci/console/job_order.rb +5 -3
- data/lib/fun_ci/console/job_row.rb +17 -0
- data/lib/fun_ci/console/job_rows.rb +14 -3
- data/lib/fun_ci/console/key_handler.rb +88 -51
- data/lib/fun_ci/console/question.rb +63 -0
- data/lib/fun_ci/console/run_message.rb +3 -3
- data/lib/fun_ci/console/trunk_marks.rb +3 -1
- data/lib/fun_ci/console/view.rb +20 -6
- data/lib/fun_ci/evidence/about_stage.rb +1 -1
- data/lib/fun_ci/evidence/collector.rb +6 -3
- data/lib/fun_ci/evidence/outcome.rb +6 -3
- data/lib/fun_ci/jobs/job_run.rb +10 -9
- data/lib/fun_ci/jobs/locks.rb +1 -1
- data/lib/fun_ci/persistence/active_jobs.rb +2 -11
- data/lib/fun_ci/persistence/active_runs.rb +48 -16
- data/lib/fun_ci/persistence/arrival.rb +18 -0
- data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
- data/lib/fun_ci/persistence/database.rb +28 -2
- data/lib/fun_ci/persistence/died.rb +21 -0
- data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
- data/lib/fun_ci/persistence/job_recorder.rb +1 -0
- data/lib/fun_ci/persistence/job_runs.rb +17 -7
- data/lib/fun_ci/persistence/job_visibility.rb +43 -0
- data/lib/fun_ci/persistence/null_recorder.rb +4 -0
- data/lib/fun_ci/persistence/pipeline_recorder.rb +23 -3
- data/lib/fun_ci/persistence/pipeline_run.rb +25 -11
- data/lib/fun_ci/persistence/stage_job.rb +25 -0
- data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
- data/lib/fun_ci/pipeline/group_kill.rb +38 -0
- data/lib/fun_ci/pipeline/pipeline_forker.rb +3 -1
- data/lib/fun_ci/pipeline/process_runner.rb +14 -8
- data/lib/fun_ci/pipeline/process_tree.rb +37 -0
- data/lib/fun_ci/pipeline/progress_reporter.rb +2 -1
- data/lib/fun_ci/pipeline/run_canceller.rb +16 -6
- data/lib/fun_ci/pipeline/slot.rb +12 -0
- data/lib/fun_ci/pipeline/slot_run.rb +20 -5
- data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
- data/lib/fun_ci/pipeline/stage_end.rb +2 -1
- data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
- data/lib/fun_ci/pipeline/trigger.rb +9 -1
- data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
- data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
- data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
- data/lib/fun_ci/pipeline/workspaces.rb +2 -1
- data/lib/fun_ci/pipeline/worktree_pool.rb +27 -7
- data/lib/fun_ci/pipeline/worktree_prune.rb +9 -4
- data/lib/fun_ci/pipeline/worktrees.rb +41 -7
- data/lib/fun_ci/setup/commands.rb +16 -7
- data/lib/fun_ci/setup/eslint_detection.rb +38 -0
- data/lib/fun_ci/setup/init_options.rb +34 -0
- data/lib/fun_ci/setup/installer.rb +41 -7
- data/lib/fun_ci/setup/placeholders.rb +30 -0
- data/lib/fun_ci/setup/project_config.rb +1 -0
- data/lib/fun_ci/setup/settings.rb +21 -1
- data/lib/fun_ci/setup/setup_checker.rb +5 -3
- data/lib/fun_ci/setup/stage_templates/javascript.rb +14 -4
- data/lib/fun_ci/setup/stage_templates/placeholders.rb +21 -0
- data/lib/fun_ci/setup/stage_templates.rb +6 -4
- data/lib/fun_ci/setup/template_writer.rb +3 -5
- data/lib/fun_ci/trunk/check.rb +7 -2
- data/lib/fun_ci/version.rb +1 -1
- data/libexec/fun-ci-renderer +0 -0
- metadata +33 -10
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Persistence
|
|
7
|
+
# Which daily and weekly jobs the console keeps out of view, and how
|
|
8
|
+
# (design.md, The console): `ignored`, until it is shown again, or
|
|
9
|
+
# `snoozed`, until the job runs again. It says nothing of whether a job
|
|
10
|
+
# runs, and the agent commands don't read it.
|
|
11
|
+
class JobVisibility
|
|
12
|
+
# When a run began, if it did: not while it waits its turn, nor when
|
|
13
|
+
# cancelled before it (design.md, Daily and weekly jobs), and once it
|
|
14
|
+
# did, whatever became of it. fun-ci 2.4.0 and earlier kept no
|
|
15
|
+
# `began_at`, so of their runs those that ran or are running began when they started.
|
|
16
|
+
BEGAN = "COALESCE(run.began_at, CASE WHEN run.status IN ('running', 'completed', 'failed', 'timed_out') " \
|
|
17
|
+
"THEN run.started_at END)"
|
|
18
|
+
HIDDEN = <<~SQL.freeze
|
|
19
|
+
SELECT job, kind FROM hidden_jobs AS hidden WHERE project_path = ? AND (kind = 'ignored' OR NOT EXISTS (
|
|
20
|
+
SELECT 1 FROM job_runs AS run WHERE run.project_path = hidden.project_path AND run.job = hidden.job
|
|
21
|
+
AND #{BEGAN} > hidden.hidden_at))
|
|
22
|
+
SQL
|
|
23
|
+
|
|
24
|
+
def initialize(db)
|
|
25
|
+
@db = db
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Keeps the project's job `name` out of view from `at`, as `kind`:
|
|
29
|
+
# `snoozed` or `ignored`. The latest of two decides.
|
|
30
|
+
def hide(project, name, kind, at:)
|
|
31
|
+
@db.execute("INSERT OR REPLACE INTO hidden_jobs (project_path, job, kind, hidden_at) VALUES (?, ?, ?, ?)",
|
|
32
|
+
[project, name, kind, at.utc.iso8601(3)])
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def show(project, name)
|
|
36
|
+
@db.execute("DELETE FROM hidden_jobs WHERE project_path = ? AND job = ?", [project, name])
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# { name => kind } of the project's jobs out of view now.
|
|
40
|
+
def hidden(project) = @db.execute(HIDDEN, [project]).to_h
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -8,6 +8,9 @@ module FunCi
|
|
|
8
8
|
class NullRecorder
|
|
9
9
|
def create_run(**); end
|
|
10
10
|
def start_stage(_stage, **); end
|
|
11
|
+
def schedule_stage(_stage, **); end
|
|
12
|
+
def started_in_turn?(_job_id) = true
|
|
13
|
+
def arrival_place = 0
|
|
11
14
|
def end_stage(_job_id, _status); end
|
|
12
15
|
def keep_exit(_job_id, _exit_status, _signal); end
|
|
13
16
|
def keep_evidence(_job_id, _document); end
|
|
@@ -19,6 +22,7 @@ module FunCi
|
|
|
19
22
|
def trunk_heads(**) = {}
|
|
20
23
|
def trunk_fetches = TrunkRecording::NoFetches.new
|
|
21
24
|
def alongside(_job_id) = []
|
|
25
|
+
def alongside_elsewhere(_job_id) = []
|
|
22
26
|
def stage_process(_job_id, _pid); end
|
|
23
27
|
def slot_taken(_lock_file); end
|
|
24
28
|
def foreground_done; end
|
|
@@ -71,16 +71,31 @@ module FunCi
|
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
def start_stage(stage, budget: nil)
|
|
74
|
+
schedule_stage(stage, budget: budget).tap { |job_id| stage_started(job_id) if job_id }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# A stage that waits before it starts, as a slow suite does for its turn.
|
|
78
|
+
def schedule_stage(stage, budget: nil)
|
|
74
79
|
return nil unless @pipeline_run_id
|
|
75
80
|
|
|
76
81
|
tolerating do
|
|
77
82
|
ensure_running
|
|
78
|
-
|
|
79
|
-
StageJob.update_status(@db, job_id, "running")
|
|
80
|
-
job_id
|
|
83
|
+
StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
|
|
81
84
|
end
|
|
82
85
|
end
|
|
83
86
|
|
|
87
|
+
# Records a slow suite started once its turn comes; false when its run
|
|
88
|
+
# was cancelled while it waited. One whose stage could not be recorded,
|
|
89
|
+
# or whose start the database refused, starts.
|
|
90
|
+
def started_in_turn?(job_id)
|
|
91
|
+
return true unless job_id
|
|
92
|
+
|
|
93
|
+
tolerating { StageJob.started_in_turn?(@db, job_id) } != false
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Where this run's commit arrived on its branch (Persistence::Arrival); 0 when not known.
|
|
97
|
+
def arrival_place = (@pipeline_run_id && tolerating { PipelineRun.arrival_place(@db, @pipeline_run_id) }) || 0
|
|
98
|
+
|
|
84
99
|
# Keeps a failed stage's evidence, and the output's tail where an older
|
|
85
100
|
# fun-ci sharing the database reads it.
|
|
86
101
|
def keep_evidence(job_id, document)
|
|
@@ -97,6 +112,7 @@ module FunCi
|
|
|
97
112
|
|
|
98
113
|
# The stages that shared the slot with this one (architecture.md, "Evidence of a failed stage").
|
|
99
114
|
def alongside(job_id) = tolerating { StageJob.alongside(@db, job_id) } || []
|
|
115
|
+
def alongside_elsewhere(job_id) = tolerating { StageJob.alongside_elsewhere(@db, job_id) } || []
|
|
100
116
|
|
|
101
117
|
def keep_exit(job_id, exit_status, signal)
|
|
102
118
|
tolerating { StageJob.keep_exit(@db, job_id, exit_status, signal) }
|
|
@@ -112,6 +128,10 @@ module FunCi
|
|
|
112
128
|
|
|
113
129
|
private
|
|
114
130
|
|
|
131
|
+
def stage_started(job_id)
|
|
132
|
+
tolerating { StageJob.update_status(@db, job_id, "running") }
|
|
133
|
+
end
|
|
134
|
+
|
|
115
135
|
def ensure_running
|
|
116
136
|
run = PipelineRun.find(@db, @pipeline_run_id)
|
|
117
137
|
return unless run && run[:status] == "scheduled"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "time"
|
|
4
|
+
require_relative "arrival"
|
|
4
5
|
|
|
5
6
|
module FunCi
|
|
6
7
|
module Persistence
|
|
@@ -56,9 +57,13 @@ module FunCi
|
|
|
56
57
|
db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
|
|
57
58
|
end
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
db.
|
|
60
|
+
# The id of the first run of this run's commit on its branch: where the commit arrived.
|
|
61
|
+
def self.arrival_place(db, id)
|
|
62
|
+
db.get_first_value("SELECT #{Arrival::FIRST_RUN} FROM pipeline_runs WHERE id = ?", [id])
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.update_status(db, id, new_status, at: Time.now)
|
|
66
|
+
db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, at.utc.iso8601, id])
|
|
62
67
|
end
|
|
63
68
|
|
|
64
69
|
def self.recent(db, limit: 20)
|
|
@@ -69,16 +74,25 @@ module FunCi
|
|
|
69
74
|
NEEDING = "CASE status WHEN 'failed' THEN 0 WHEN 'timed_out' THEN 1 WHEN 'running' THEN 2 " \
|
|
70
75
|
"WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
|
|
71
76
|
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
+
# Each branch's newest run.
|
|
78
|
+
# Each branch's head: the newest run of the commit that arrived on it last (Arrival).
|
|
79
|
+
HEADS = "SELECT MAX(id) FROM pipeline_runs WHERE #{Arrival::FIRST_RUN} IN " \
|
|
80
|
+
"(SELECT MAX(#{Arrival::FIRST_RUN}) FROM pipeline_runs GROUP BY project_path, branch) " \
|
|
81
|
+
"GROUP BY project_path, branch".freeze
|
|
82
|
+
|
|
83
|
+
# The newest run of each of `limit` branches, of the `among` whose newest
|
|
84
|
+
# run changed most recently: those that most need you, then those run
|
|
85
|
+
# most recently, in that order.
|
|
86
|
+
def self.branch_heads(db, limit:, among:)
|
|
87
|
+
query(db, "WHERE id IN (SELECT id FROM pipeline_runs WHERE id IN (#{HEADS}) " \
|
|
88
|
+
"ORDER BY updated_at DESC, id DESC LIMIT ?) ORDER BY #{NEEDING}, id DESC LIMIT ?", among, limit)
|
|
77
89
|
end
|
|
78
90
|
|
|
79
|
-
#
|
|
80
|
-
|
|
81
|
-
|
|
91
|
+
# The runs of `head`'s branch up to it, newest first: not a run of an
|
|
92
|
+
# older commit run again since.
|
|
93
|
+
def self.leading_to(db, head, limit:)
|
|
94
|
+
query(db, "WHERE project_path IS ? AND branch = ? AND id <= ? ORDER BY id DESC LIMIT ?",
|
|
95
|
+
head[:project_path], head[:branch], head[:id], limit)
|
|
82
96
|
end
|
|
83
97
|
|
|
84
98
|
def self.query(db, clause, *params)
|
|
@@ -22,6 +22,19 @@ module FunCi
|
|
|
22
22
|
db.last_insert_row_id
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
+
# Records a slow suite waiting its branch's turn running, unless it or its
|
|
26
|
+
# run was recorded cancelled meanwhile; whether it was. One statement, so
|
|
27
|
+
# a cancel recorded before it refuses it and one after finds it running
|
|
28
|
+
# (acceptance-tests.md, AT-9.19).
|
|
29
|
+
def self.started_in_turn?(db, id)
|
|
30
|
+
db.execute("UPDATE stage_jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'scheduled' " \
|
|
31
|
+
"AND NOT EXISTS (#{RUN_CANCELLED})", [Time.now.utc.iso8601(3), id])
|
|
32
|
+
db.changes == 1
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
RUN_CANCELLED = "SELECT 1 FROM pipeline_runs WHERE pipeline_runs.id = stage_jobs.pipeline_run_id " \
|
|
36
|
+
"AND pipeline_runs.status = 'cancelled'"
|
|
37
|
+
|
|
25
38
|
# The stage script's process, which leads a process group of its own.
|
|
26
39
|
def self.store_pid(db, id, pid)
|
|
27
40
|
db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
|
|
@@ -51,6 +64,18 @@ module FunCi
|
|
|
51
64
|
[id]).flatten
|
|
52
65
|
end
|
|
53
66
|
|
|
67
|
+
# Stages of the project's other runs running at any moment between this
|
|
68
|
+
# stage's start and now, as "<short commit>'s <stage>".
|
|
69
|
+
def self.alongside_elsewhere(db, id)
|
|
70
|
+
db.execute("SELECT runs.commit_hash, others.stage FROM stage_jobs AS others " \
|
|
71
|
+
"JOIN pipeline_runs AS runs ON runs.id = others.pipeline_run_id " \
|
|
72
|
+
"JOIN stage_jobs AS me ON me.id = ? JOIN pipeline_runs AS mine ON mine.id = me.pipeline_run_id " \
|
|
73
|
+
"WHERE runs.project_path IS mine.project_path AND runs.id != mine.id " \
|
|
74
|
+
"AND others.started_at IS NOT NULL " \
|
|
75
|
+
"AND (others.completed_at IS NULL OR others.completed_at > me.started_at) ORDER BY others.id",
|
|
76
|
+
[id]).map { |sha, stage| "#{sha[0, 7]}'s #{stage}" }
|
|
77
|
+
end
|
|
78
|
+
|
|
54
79
|
def self.ids(db) = db.execute("SELECT id FROM stage_jobs").flatten
|
|
55
80
|
|
|
56
81
|
def self.find(db, id)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "time"
|
|
5
5
|
require_relative "../trunk/check"
|
|
6
|
+
require_relative "pipeline_run"
|
|
6
7
|
|
|
7
8
|
module FunCi
|
|
8
9
|
module Persistence
|
|
@@ -12,7 +13,7 @@ module FunCi
|
|
|
12
13
|
TIP = %i[trunk_remote trunk_branch trunk_sha trunk_seen_at].freeze
|
|
13
14
|
MERGE = %i[outcome ahead behind files reason].freeze
|
|
14
15
|
COLUMNS = [:commit_hash, *TIP, *MERGE].freeze
|
|
15
|
-
SELECT = "SELECT #{COLUMNS.join(", ")} FROM trunk_checks".freeze
|
|
16
|
+
SELECT = "SELECT #{COLUMNS.join(", ")}, checked_at FROM trunk_checks".freeze
|
|
16
17
|
|
|
17
18
|
def initialize(db, project)
|
|
18
19
|
@db = db
|
|
@@ -36,14 +37,14 @@ module FunCi
|
|
|
36
37
|
found = @db.execute("#{SELECT} WHERE project_path = ? AND commit_hash = ? " \
|
|
37
38
|
"ORDER BY COALESCE(trunk_seen_at, checked_at) DESC, id DESC LIMIT 1", [@project, sha]).first
|
|
38
39
|
found && Trunk::Check.new(commit: found.first, tip: tip(found[1, TIP.size]),
|
|
39
|
-
merge: merge(found.
|
|
40
|
+
merge: merge(found[1 + TIP.size, MERGE.size]), checked_at: Time.parse(found.last))
|
|
40
41
|
end
|
|
41
42
|
|
|
42
|
-
# { commit => trunk SHA } for each branch's
|
|
43
|
-
# whose commit has been checked against a trunk tip.
|
|
43
|
+
# { commit => trunk SHA } for each branch's head (PipelineRun::HEADS), other
|
|
44
|
+
# than `except`, whose commit has been checked against a trunk tip.
|
|
44
45
|
def branch_heads(except:)
|
|
45
|
-
heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (
|
|
46
|
-
"
|
|
46
|
+
heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (#{PipelineRun::HEADS}) " \
|
|
47
|
+
"AND project_path = ? AND commit_hash != ?", [@project, except]).flatten
|
|
47
48
|
heads.to_h { |sha| [sha, latest(sha)&.tip&.sha] }.compact
|
|
48
49
|
end
|
|
49
50
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Pipeline
|
|
5
|
+
# Kills a process group and what left it (ProcessTree): TERM, then KILL.
|
|
6
|
+
# A member forking as the KILL arrives leaves a child in the group that
|
|
7
|
+
# the signal missed, so the group is killed again while any member lives,
|
|
8
|
+
# a bounded number of times, since a mutant may stop it ever going.
|
|
9
|
+
module GroupKill
|
|
10
|
+
ROUNDS = 10
|
|
11
|
+
|
|
12
|
+
# escaped: further targets, -pgid for a group, pid for a process.
|
|
13
|
+
def self.kill(pid, escaped, signal: Process.method(:kill))
|
|
14
|
+
%w[TERM KILL].product([-pid, *escaped]).each { |name, target| send_to(signal, name, target) }
|
|
15
|
+
ROUNDS.times do
|
|
16
|
+
return unless lives?(pid, signal)
|
|
17
|
+
|
|
18
|
+
send_to(signal, "KILL", -pid)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# On macOS a group left with only zombies answers EPERM, not ESRCH.
|
|
23
|
+
def self.lives?(pid, signal)
|
|
24
|
+
signal.call(0, -pid)
|
|
25
|
+
true
|
|
26
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
27
|
+
false
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.send_to(signal, name, target)
|
|
31
|
+
signal.call(name, target)
|
|
32
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
33
|
+
# Gone already.
|
|
34
|
+
end
|
|
35
|
+
private_class_method :lives?, :send_to
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -66,8 +66,10 @@ module FunCi
|
|
|
66
66
|
end
|
|
67
67
|
|
|
68
68
|
# Trigger requires this file, through TriggerCommand, so it is loaded
|
|
69
|
-
# here, where it is used, rather than above.
|
|
69
|
+
# here, where it is used, rather than above. The run leads a session of
|
|
70
|
+
# its own, so that ending the shell that committed doesn't end it.
|
|
70
71
|
def self.run_in_child(commit_hash:, branch:, db_path:)
|
|
72
|
+
Process.setsid
|
|
71
73
|
require_relative "trigger"
|
|
72
74
|
recorder = Persistence::DbRecorder.new(Persistence::Database.connection(db_path))
|
|
73
75
|
trigger = trigger(commit_hash, branch, recorder)
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
require_relative "gate"
|
|
4
4
|
require_relative "git_environment"
|
|
5
5
|
require_relative "output_window"
|
|
6
|
+
require_relative "process_tree"
|
|
7
|
+
require_relative "group_kill"
|
|
8
|
+
require_relative "../evidence/process_table"
|
|
6
9
|
|
|
7
10
|
module FunCi
|
|
8
11
|
module Pipeline
|
|
@@ -26,13 +29,14 @@ module FunCi
|
|
|
26
29
|
# fun-ci's own, the window what it prints is written to, what to do
|
|
27
30
|
# with its pid when it runs over budget, before it is killed, the
|
|
28
31
|
# open files it holds as long as it runs, such as a lock, and what it
|
|
29
|
-
# is run under (a Priorities prefix, "" for fun-ci's own priority)
|
|
30
|
-
|
|
32
|
+
# is run under (a Priorities prefix, "" for fun-ci's own priority), and
|
|
33
|
+
# how long its output may drain once it is killed.
|
|
34
|
+
Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority, :drain)
|
|
31
35
|
|
|
32
36
|
class Launch
|
|
33
37
|
def initialize(**given)
|
|
34
38
|
super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
|
|
35
|
-
**given)
|
|
39
|
+
drain: DRAIN_SECONDS, **given)
|
|
36
40
|
end
|
|
37
41
|
end
|
|
38
42
|
|
|
@@ -97,12 +101,12 @@ module FunCi
|
|
|
97
101
|
def over_budget(pid, reading, launch)
|
|
98
102
|
launch.before_kill&.call(pid)
|
|
99
103
|
kill_process_group(pid)
|
|
100
|
-
[drained(reading, launch
|
|
104
|
+
[drained(reading, launch), nil, true]
|
|
101
105
|
end
|
|
102
106
|
|
|
103
|
-
def drained(reading,
|
|
104
|
-
reading.join(
|
|
105
|
-
text(
|
|
107
|
+
def drained(reading, launch)
|
|
108
|
+
reading.join(launch.drain)
|
|
109
|
+
text(launch.output)
|
|
106
110
|
end
|
|
107
111
|
|
|
108
112
|
def text(window) = window.text.force_encoding(Encoding.default_external)
|
|
@@ -112,8 +116,10 @@ module FunCi
|
|
|
112
116
|
[output, status, false]
|
|
113
117
|
end
|
|
114
118
|
|
|
119
|
+
# With the group go what it started that left it (ProcessTree), found
|
|
120
|
+
# before the kill orphans them (GroupKill).
|
|
115
121
|
def kill_process_group(pid)
|
|
116
|
-
|
|
122
|
+
GroupKill.kill(pid, ProcessTree.escaped([pid], Evidence::ProcessTable.now))
|
|
117
123
|
ignoring_errors { Process.waitpid(pid) }
|
|
118
124
|
end
|
|
119
125
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Pipeline
|
|
5
|
+
# What to kill besides a stage's process groups: each process descended
|
|
6
|
+
# from one of their members, by parent id, that has left for a group or
|
|
7
|
+
# session of its own, as Node's detached children do. The rows must be
|
|
8
|
+
# read before the kill, since a killed parent's children move to init.
|
|
9
|
+
# A double-forked daemon has moved to init already and is not reached.
|
|
10
|
+
module ProcessTree
|
|
11
|
+
# Signal targets: -pid for an escapee leading its own group, which
|
|
12
|
+
# takes that group with it, and the pid alone of one in any other group.
|
|
13
|
+
def self.escaped(groups, rows)
|
|
14
|
+
escapees = outside(groups, descendants(in_groups(groups, rows), rows))
|
|
15
|
+
leaders = escapees.select { |row| row.pgid == row.pid }.map(&:pid)
|
|
16
|
+
leaders.map(&:-@) + escapees.reject { |row| leaders.include?(row.pgid) }.map(&:pid)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.in_groups(groups, rows) = rows.select { |row| groups.include?(row.pgid) }
|
|
20
|
+
def self.outside(groups, rows) = rows.reject { |row| groups.include?(row.pgid) }
|
|
21
|
+
|
|
22
|
+
# Bounded by the table's size, so a table whose parent ids form a cycle still ends.
|
|
23
|
+
def self.descendants(found, rows)
|
|
24
|
+
rows.size.times do
|
|
25
|
+
more = children(found.map(&:pid), rows) - found
|
|
26
|
+
return found if more.empty?
|
|
27
|
+
|
|
28
|
+
found += more
|
|
29
|
+
end
|
|
30
|
+
found
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.children(parents, rows) = rows.select { |row| parents.include?(row.ppid) }
|
|
34
|
+
private_class_method :in_groups, :outside, :descendants, :children
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -18,7 +18,8 @@ module FunCi
|
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
def cold_slot(seconds)
|
|
21
|
-
@stdout.puts "fun-ci: a
|
|
21
|
+
@stdout.puts "fun-ci: a cold worktree (new, just cleaned, or unused for a day), so lint and build have " \
|
|
22
|
+
"#{seconds}s this once."
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
def slow_launched
|
|
@@ -5,18 +5,24 @@ require_relative "../persistence/active_jobs"
|
|
|
5
5
|
require_relative "../persistence/run_status"
|
|
6
6
|
require_relative "../persistence/write_trouble"
|
|
7
7
|
require_relative "slot_lock"
|
|
8
|
+
require_relative "process_tree"
|
|
9
|
+
require_relative "../evidence/process_table"
|
|
8
10
|
|
|
9
11
|
module FunCi
|
|
10
12
|
module Pipeline
|
|
11
13
|
# Stops every process of a run. The run's own processes go first, so none
|
|
12
14
|
# of them records the run as failed once its stages die; then each stage's
|
|
13
|
-
# process group
|
|
15
|
+
# process group, and last what a stage started that left its group
|
|
16
|
+
# (ProcessTree). A run whose slot nobody holds any more has died already,
|
|
14
17
|
# and the pids it recorded may belong to other processes now, so it is
|
|
15
18
|
# left alone; so is this process.
|
|
16
19
|
class RunCanceller
|
|
17
|
-
|
|
20
|
+
# process_table: what ps lists now, as Evidence::ProcessTable rows.
|
|
21
|
+
def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?),
|
|
22
|
+
process_table: Evidence::ProcessTable.method(:now))
|
|
18
23
|
@killer = killer
|
|
19
24
|
@slot_held = slot_held
|
|
25
|
+
@process_table = process_table
|
|
20
26
|
end
|
|
21
27
|
|
|
22
28
|
# Stops the run and records it, and its unfinished stages, cancelled.
|
|
@@ -60,18 +66,22 @@ module FunCi
|
|
|
60
66
|
def stop(run)
|
|
61
67
|
return unless alive?(run)
|
|
62
68
|
|
|
63
|
-
(run.processes - [Process.pid] + run.stage_groups.map(&:-@)).each { |pid| kill(pid) }
|
|
69
|
+
(run.processes - [Process.pid] + run.stage_groups.map(&:-@) + escaped(run)).each { |pid| kill(pid) }
|
|
64
70
|
end
|
|
65
71
|
|
|
66
72
|
private
|
|
67
73
|
|
|
74
|
+
def escaped(run)
|
|
75
|
+
return [] if run.stage_groups.empty?
|
|
76
|
+
|
|
77
|
+
ProcessTree.escaped(run.stage_groups, @process_table.call)
|
|
78
|
+
end
|
|
79
|
+
|
|
68
80
|
# A stage its process started after `run` was read. A run's process
|
|
69
81
|
# records a stage's group before it lets the stage start, so once those
|
|
70
82
|
# processes are dead, looking again finds every stage they started.
|
|
71
83
|
def stop_late_stages(db, run)
|
|
72
|
-
Persistence::ActiveRuns.
|
|
73
|
-
(now.stage_groups - run.stage_groups).each { |group| kill(-group) }
|
|
74
|
-
end
|
|
84
|
+
(Persistence::ActiveRuns.groups(db, run.id) - run.stage_groups).each { |group| kill(-group) }
|
|
75
85
|
end
|
|
76
86
|
|
|
77
87
|
# The groups recorded since `before` was read.
|
data/lib/fun_ci/pipeline/slot.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "waiting_slots"
|
|
4
|
+
|
|
3
5
|
module FunCi
|
|
4
6
|
module Pipeline
|
|
5
7
|
# A worktree a pipeline has to itself, held through an flock on its lock
|
|
@@ -26,6 +28,16 @@ module FunCi
|
|
|
26
28
|
@lock.close if @holders.zero? && !@lock.closed?
|
|
27
29
|
end
|
|
28
30
|
|
|
31
|
+
# Marks the slot as held only by a slow suite waiting its turn, until
|
|
32
|
+
# the mark is closed (WaitingSlots). A slot with nothing to lock has
|
|
33
|
+
# nothing to mark.
|
|
34
|
+
def waiting = lock_file ? WaitingSlots.mark(path) : Unmarked.new
|
|
35
|
+
|
|
36
|
+
# The mark of a slot with nothing to lock: closing it does nothing.
|
|
37
|
+
class Unmarked
|
|
38
|
+
def close; end
|
|
39
|
+
end
|
|
40
|
+
|
|
29
41
|
# Whether its worktree was just made, so has no caches yet.
|
|
30
42
|
def cold? = false
|
|
31
43
|
|
|
@@ -5,6 +5,7 @@ require_relative "background_fork"
|
|
|
5
5
|
require_relative "stage_runner"
|
|
6
6
|
require_relative "stage_execution"
|
|
7
7
|
require_relative "progress_reporter"
|
|
8
|
+
require_relative "turn_queue"
|
|
8
9
|
|
|
9
10
|
module FunCi
|
|
10
11
|
module Pipeline
|
|
@@ -71,26 +72,40 @@ module FunCi
|
|
|
71
72
|
end
|
|
72
73
|
|
|
73
74
|
def launch_slow_suite(config)
|
|
74
|
-
job_id = recorder.
|
|
75
|
+
job_id = recorder.schedule_stage("slow", budget: @seams.budgets["slow"])
|
|
75
76
|
launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
|
|
76
77
|
executor: slow_suite(config, @slot.share))
|
|
77
78
|
end
|
|
78
79
|
|
|
79
80
|
# Called with the recorder and job to record the slow suite with, in
|
|
80
|
-
# whichever process runs it
|
|
81
|
+
# whichever process runs it, once no older slow suite of the branch runs
|
|
82
|
+
# (acceptance-tests.md, AT-9.19); the slot is let go once it is recorded.
|
|
81
83
|
def slow_suite(config, slot)
|
|
82
84
|
execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
|
|
83
85
|
launching: { priority: @seams.priorities.slow })
|
|
84
86
|
command = config.stage_command("slow", @commit.sha)
|
|
85
|
-
->(recorder, job_id) {
|
|
87
|
+
->(recorder, job_id) { in_turn(slot, recorder, job_id) { execution.run("slow", command, recorder, job_id) } }
|
|
86
88
|
end
|
|
87
89
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
# Holding the slot until the slow suite is recorded, whether or not its
|
|
91
|
+
# turn came, marked waiting until it does (WaitingSlots); a run
|
|
92
|
+
# cancelled while it waited never starts it.
|
|
93
|
+
def in_turn(slot, recorder, job_id, &)
|
|
94
|
+
waiting = slot.waiting
|
|
95
|
+
take_turn(waiting, recorder, job_id, &)
|
|
90
96
|
ensure
|
|
97
|
+
waiting&.close
|
|
91
98
|
slot.release
|
|
92
99
|
end
|
|
93
100
|
|
|
101
|
+
def take_turn(waiting, recorder, job_id)
|
|
102
|
+
ticket = TurnTicket.new(place: recorder.arrival_place, id: job_id.to_i)
|
|
103
|
+
@seams.slow_turns.take(@commit.branch, ticket) do
|
|
104
|
+
waiting.close
|
|
105
|
+
yield if recorder.started_in_turn?(job_id)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
94
109
|
def launch(**)
|
|
95
110
|
return @seams.background_launcher.call(**) if @seams.background_launcher
|
|
96
111
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require_relative "turn_queue"
|
|
6
|
+
|
|
7
|
+
module FunCi
|
|
8
|
+
module Pipeline
|
|
9
|
+
# The turns a branch's slow suites take: one runs at a time, and of those
|
|
10
|
+
# waiting, the one whose commit arrived last goes next (acceptance-tests.md,
|
|
11
|
+
# AT-9.19, AT-9.20). A turn is an flock on the branch's file under `root`,
|
|
12
|
+
# and each suite holds a ticket in the branch's TurnQueue while it waits and
|
|
13
|
+
# runs, so both end with the process that held them, a slow suite cancelled
|
|
14
|
+
# while it waits included.
|
|
15
|
+
class SlowTurns
|
|
16
|
+
# The most passes taking a turn makes. Each but the last lets a newer
|
|
17
|
+
# suite run to its end, so this covers more than a branch queues; it
|
|
18
|
+
# stops a pass that never ends, as a mutant's, all the same.
|
|
19
|
+
ROUNDS = 100
|
|
20
|
+
|
|
21
|
+
# wait: called once each pass with the paths of the newer suites'
|
|
22
|
+
# tickets to let go first, none on the last; blocks until those are done.
|
|
23
|
+
def initialize(root, wait: TurnQueue::WAIT)
|
|
24
|
+
@root = root
|
|
25
|
+
@wait = wait
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Yields once no other process holds the branch's turn and no newer
|
|
29
|
+
# suite of the branch waits for it, holding the turn until the block returns.
|
|
30
|
+
def take(branch, ticket)
|
|
31
|
+
queue = TurnQueue.new(queue_dir(branch))
|
|
32
|
+
queue.holding(ticket) do
|
|
33
|
+
File.open(lock_path(branch), File::RDWR | File::CREAT, 0o644) do |lock|
|
|
34
|
+
in_order(queue, ticket, lock)
|
|
35
|
+
yield
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def lock_path(branch) = File.join(@root, "#{digest(branch)}.lock")
|
|
41
|
+
def ticket_path(branch, ticket) = File.join(queue_dir(branch), ticket.name)
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
# Each newer suite goes first; with the turn held it looks again, as a
|
|
46
|
+
# newer one may have come meanwhile. A wait ends only once its ticket's
|
|
47
|
+
# holder let go, so each ticket is waited for once: one that still seems
|
|
48
|
+
# held is no live suite's, and the passes end with the tickets seen.
|
|
49
|
+
def in_order(queue, ticket, lock)
|
|
50
|
+
waited = Set.new
|
|
51
|
+
ROUNDS.times do
|
|
52
|
+
waited.merge(let_go(queue.newer_than(ticket) - waited))
|
|
53
|
+
return if turn_taken?(queue, ticket, lock, waited)
|
|
54
|
+
end
|
|
55
|
+
lock.flock(File::LOCK_EX)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Takes the turn, and whether no suite newer than those `waited` for
|
|
59
|
+
# came meanwhile; lets the turn go again if one did.
|
|
60
|
+
def turn_taken?(queue, ticket, lock, waited)
|
|
61
|
+
lock.flock(File::LOCK_EX)
|
|
62
|
+
return true if (queue.newer_than(ticket) - waited).empty?
|
|
63
|
+
|
|
64
|
+
lock.flock(File::LOCK_UN)
|
|
65
|
+
false
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def let_go(tickets) = tickets.tap { @wait.call(tickets) }
|
|
69
|
+
|
|
70
|
+
def queue_dir(branch) = File.join(@root, digest(branch))
|
|
71
|
+
def digest(branch) = Digest::SHA256.hexdigest(branch)[0, 16]
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -58,7 +58,8 @@ module FunCi
|
|
|
58
58
|
status = finished.status
|
|
59
59
|
Evidence::Outcome.new(state: STATES.fetch(outcome(finished)), exit_status: status&.exitstatus,
|
|
60
60
|
signal: status&.termsig && Signal.signame(status.termsig),
|
|
61
|
-
alongside: @recorder.alongside(@job_id),
|
|
61
|
+
alongside: @recorder.alongside(@job_id),
|
|
62
|
+
elsewhere: @recorder.alongside_elsewhere(@job_id), overrun: finished.overrun)
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
def keep_exit(status)
|
|
@@ -21,7 +21,7 @@ module FunCi
|
|
|
21
21
|
|
|
22
22
|
def cancel(new_commit_hash:)
|
|
23
23
|
waited_before = (Time.now - WAITED_ON_SECONDS).utc.iso8601
|
|
24
|
-
Persistence::ActiveRuns.
|
|
24
|
+
Persistence::ActiveRuns.cancel_stale(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
|
|
25
25
|
.each { |run| cancel_run(run, new_commit_hash) }
|
|
26
26
|
end
|
|
27
27
|
|