fun_ci 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/README.md +16 -7
- data/lib/fun_ci/agent/events.rb +15 -7
- data/lib/fun_ci/agent/job_events.rb +12 -5
- data/lib/fun_ci/agent/onward.rb +21 -0
- data/lib/fun_ci/agent/output.rb +7 -0
- data/lib/fun_ci/agent/reports.rb +3 -0
- data/lib/fun_ci/agent/snapshots.rb +25 -7
- data/lib/fun_ci/agent/status_text.rb +7 -3
- data/lib/fun_ci/agent/wait_command.rb +37 -5
- data/lib/fun_ci/console/board_data.rb +46 -11
- data/lib/fun_ci/console/branch_page.rb +9 -0
- data/lib/fun_ci/console/console_state.rb +22 -8
- data/lib/fun_ci/console/cursor.rb +22 -0
- data/lib/fun_ci/console/cursor_rows.rb +37 -0
- data/lib/fun_ci/console/job_message.rb +5 -4
- data/lib/fun_ci/console/job_order.rb +5 -3
- data/lib/fun_ci/console/job_row.rb +17 -0
- data/lib/fun_ci/console/job_rows.rb +14 -3
- data/lib/fun_ci/console/key_handler.rb +88 -51
- data/lib/fun_ci/console/question.rb +63 -0
- data/lib/fun_ci/console/run_message.rb +3 -3
- data/lib/fun_ci/console/view.rb +20 -6
- data/lib/fun_ci/jobs/job_run.rb +10 -9
- data/lib/fun_ci/persistence/active_runs.rb +47 -17
- data/lib/fun_ci/persistence/arrival.rb +18 -0
- data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
- data/lib/fun_ci/persistence/database.rb +28 -2
- data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
- data/lib/fun_ci/persistence/job_runs.rb +17 -7
- data/lib/fun_ci/persistence/job_visibility.rb +43 -0
- data/lib/fun_ci/persistence/null_recorder.rb +3 -0
- data/lib/fun_ci/persistence/pipeline_recorder.rb +22 -3
- data/lib/fun_ci/persistence/pipeline_run.rb +15 -4
- data/lib/fun_ci/persistence/stage_job.rb +13 -0
- data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
- data/lib/fun_ci/pipeline/process_runner.rb +8 -7
- data/lib/fun_ci/pipeline/run_canceller.rb +1 -3
- data/lib/fun_ci/pipeline/slot.rb +12 -0
- data/lib/fun_ci/pipeline/slot_run.rb +20 -5
- data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
- data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
- data/lib/fun_ci/pipeline/trigger.rb +9 -1
- data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
- data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
- data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
- data/lib/fun_ci/pipeline/worktree_pool.rb +5 -2
- data/lib/fun_ci/pipeline/worktrees.rb +3 -0
- data/lib/fun_ci/trunk/check.rb +7 -2
- data/lib/fun_ci/version.rb +1 -1
- metadata +26 -11
|
@@ -4,6 +4,7 @@ require "time"
|
|
|
4
4
|
require_relative "pipeline_run"
|
|
5
5
|
require_relative "stage_job"
|
|
6
6
|
require_relative "died"
|
|
7
|
+
require_relative "arrival"
|
|
7
8
|
|
|
8
9
|
module FunCi
|
|
9
10
|
module Persistence
|
|
@@ -20,31 +21,49 @@ module FunCi
|
|
|
20
21
|
module ActiveRuns
|
|
21
22
|
ACTIVE = "status IN ('scheduled', 'running')"
|
|
22
23
|
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
# Records cancelled, and returns, the runs on the branch a new commit's
|
|
25
|
+
# run cancels: those of commits that arrived before it (acceptance-tests.md,
|
|
26
|
+
# AT-9.20), except one an agent has waited on since `waited_before` (an
|
|
27
|
+
# ISO 8601 UTC time; AT-9.12) and one whose slow suite is running (AT-9.19).
|
|
28
|
+
# Picked and recorded in one statement, before their processes are
|
|
29
|
+
# stopped, so a slow suite whose turn comes meanwhile never starts
|
|
30
|
+
# (StageJob.started_in_turn?).
|
|
31
|
+
def self.cancel_stale(db, branch, new_commit:, waited_before:)
|
|
32
|
+
values = [Time.now.utc.iso8601, branch.project, branch.name, branch.project, branch.name, new_commit,
|
|
33
|
+
waited_before]
|
|
34
|
+
rows = db.execute("UPDATE pipeline_runs SET status = 'cancelled', updated_at = ? WHERE #{STALE} " \
|
|
35
|
+
"RETURNING #{RUN_COLUMNS}", values)
|
|
36
|
+
rows.sort_by(&:first).map { |row| active_run(db, row) }
|
|
29
37
|
end
|
|
30
38
|
|
|
39
|
+
# A commit arrives on its branch with its first run; a commit with none yet arrives last.
|
|
40
|
+
ARRIVED_BEFORE = "#{Arrival::FIRST_RUN} < COALESCE((SELECT MIN(id) FROM pipeline_runs " \
|
|
41
|
+
"WHERE project_path IS ? AND branch = ? AND commit_hash = ?), #{Arrival::NOT_YET})".freeze
|
|
42
|
+
SLOW_RUNNING = "SELECT 1 FROM stage_jobs WHERE pipeline_run_id = pipeline_runs.id " \
|
|
43
|
+
"AND stage = 'slow' AND status = 'running'"
|
|
44
|
+
STALE = "project_path = ? AND branch = ? AND #{ARRIVED_BEFORE} AND (waited_at IS NULL OR waited_at < ?) " \
|
|
45
|
+
"AND NOT EXISTS (#{SLOW_RUNNING}) AND #{ACTIVE}".freeze
|
|
46
|
+
RUN_COLUMNS = "id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid"
|
|
47
|
+
|
|
31
48
|
# The run with this id, when it has not finished; none otherwise.
|
|
32
49
|
def self.with_id(db, id) = where(db, "id = ?", id)
|
|
33
50
|
|
|
34
|
-
# [job id, run id, pid] for each slow stage still running
|
|
35
|
-
# forked process that runs it.
|
|
51
|
+
# [job id, run id, pid] for each slow stage still running or waiting its
|
|
52
|
+
# turn (AT-9.19), pid being the forked process that runs it.
|
|
36
53
|
def self.slow_suites(db)
|
|
37
54
|
db.execute("SELECT stage_jobs.id, pipeline_runs.id, pipeline_runs.pid FROM stage_jobs " \
|
|
38
55
|
"JOIN pipeline_runs ON pipeline_runs.id = stage_jobs.pipeline_run_id " \
|
|
39
|
-
"WHERE stage_jobs.stage = 'slow' AND stage_jobs
|
|
56
|
+
"WHERE stage_jobs.stage = 'slow' AND stage_jobs.#{ACTIVE} " \
|
|
40
57
|
"AND pipeline_runs.pid IS NOT NULL")
|
|
41
58
|
end
|
|
42
59
|
|
|
43
|
-
# Records the slow stage failed,
|
|
44
|
-
# result was
|
|
60
|
+
# Records the slow stage failed, numbered after every other stage of its
|
|
61
|
+
# run that finished, and that its process died, unless its result was
|
|
62
|
+
# recorded meanwhile.
|
|
45
63
|
def self.slow_suite_died(db, job_id)
|
|
46
|
-
db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?,
|
|
47
|
-
"
|
|
64
|
+
db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?, " \
|
|
65
|
+
"finished_order = #{StageJob::NEXT_IN_RUN}, evidence = ? WHERE id = ? AND #{ACTIVE}",
|
|
66
|
+
[Time.now.utc.iso8601(3), Died.evidence("stage"), job_id])
|
|
48
67
|
end
|
|
49
68
|
|
|
50
69
|
def self.cancelled(db, run)
|
|
@@ -53,20 +72,31 @@ module FunCi
|
|
|
53
72
|
PipelineRun.update_status(db, run.id, "cancelled")
|
|
54
73
|
end
|
|
55
74
|
|
|
75
|
+
# The process groups of the run's stages still going, and its trunk
|
|
76
|
+
# fetch's, whatever the run's own status says.
|
|
77
|
+
def self.groups(db, id)
|
|
78
|
+
fetch = db.execute("SELECT fetch_pgid FROM pipeline_runs WHERE id = ?", [id]).flatten.compact
|
|
79
|
+
[*stage_groups(db, id), *fetch]
|
|
80
|
+
end
|
|
81
|
+
|
|
56
82
|
def self.where(db, clause, *values)
|
|
57
|
-
db.execute("SELECT
|
|
83
|
+
db.execute("SELECT #{RUN_COLUMNS} FROM pipeline_runs " \
|
|
58
84
|
"WHERE #{clause} AND #{ACTIVE} ORDER BY id", values).map { |row| active_run(db, row) }
|
|
59
85
|
end
|
|
60
86
|
private_class_method :where
|
|
61
87
|
|
|
62
88
|
# The groups are each running stage's, and the trunk fetch's (architecture.md, Checking against the trunk).
|
|
63
89
|
def self.active_run(db, (id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid))
|
|
64
|
-
groups = db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
|
|
65
|
-
[id]).flatten
|
|
66
90
|
ActiveRun.new(id: id, commit_hash: commit_hash, processes: [trigger_pid, pid].compact,
|
|
67
|
-
stage_groups: [*
|
|
91
|
+
stage_groups: [*stage_groups(db, id), *fetch_pgid], slot_lock: slot_lock)
|
|
68
92
|
end
|
|
69
93
|
private_class_method :active_run
|
|
94
|
+
|
|
95
|
+
def self.stage_groups(db, id)
|
|
96
|
+
db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
|
|
97
|
+
[id]).flatten
|
|
98
|
+
end
|
|
99
|
+
private_class_method :stage_groups
|
|
70
100
|
end
|
|
71
101
|
end
|
|
72
102
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Persistence
|
|
5
|
+
# When a commit arrived on its branch: its first run there. A commit run
|
|
6
|
+
# again (by a waiter, or `fun-ci trigger` by hand) keeps its place, so it
|
|
7
|
+
# supersedes no commit that arrived after it, and takes no branch's row
|
|
8
|
+
# from one (acceptance-tests.md, AT-9.20).
|
|
9
|
+
module Arrival
|
|
10
|
+
# The first run of the commit of the row `pipeline_runs` names, in SQL.
|
|
11
|
+
FIRST_RUN = "(SELECT MIN(arrived.id) FROM pipeline_runs arrived " \
|
|
12
|
+
"WHERE arrived.project_path IS pipeline_runs.project_path " \
|
|
13
|
+
"AND arrived.branch = pipeline_runs.branch AND arrived.commit_hash = pipeline_runs.commit_hash)"
|
|
14
|
+
# Later than any run: the place of a commit with no run yet.
|
|
15
|
+
NOT_YET = 9_223_372_036_854_775_807
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Persistence
|
|
7
|
+
# Which branches the console keeps out of view, and how (design.md, The
|
|
8
|
+
# console): `ignored`, until it is shown again, or `snoozed`, until a run
|
|
9
|
+
# of the branch starts after the snooze. Like JobVisibility, it says
|
|
10
|
+
# nothing of whether a branch runs, and the agent commands don't read it.
|
|
11
|
+
class BranchVisibility
|
|
12
|
+
# A run is kept to the second and a snooze to the millisecond, so they
|
|
13
|
+
# are compared as times; a run in the second of the snooze doesn't end it.
|
|
14
|
+
HIDDEN = <<~SQL
|
|
15
|
+
SELECT project_path, branch, kind FROM hidden_branches AS hidden WHERE kind = 'ignored' OR NOT EXISTS (
|
|
16
|
+
SELECT 1 FROM pipeline_runs AS run WHERE run.project_path = hidden.project_path
|
|
17
|
+
AND run.branch = hidden.branch
|
|
18
|
+
AND julianday(run.created_at) > julianday(hidden.hidden_at))
|
|
19
|
+
SQL
|
|
20
|
+
|
|
21
|
+
def initialize(db)
|
|
22
|
+
@db = db
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Keeps the project's `branch` out of view from `at`, as `kind`:
|
|
26
|
+
# `snoozed` or `ignored`. The latest of two decides.
|
|
27
|
+
def hide(project, branch, kind, at:)
|
|
28
|
+
@db.execute("INSERT OR REPLACE INTO hidden_branches (project_path, branch, kind, hidden_at) " \
|
|
29
|
+
"VALUES (?, ?, ?, ?)", [project, branch, kind, at.utc.iso8601(3)])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def show(project, branch)
|
|
33
|
+
@db.execute("DELETE FROM hidden_branches WHERE project_path = ? AND branch = ?", [project, branch])
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# { [project, branch] => kind } of the branches out of view now.
|
|
37
|
+
def hidden = @db.execute(HIDDEN).to_h { |project, branch, kind| [[project, branch], kind] }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "sqlite3"
|
|
4
|
+
require_relative "stage_job"
|
|
5
|
+
require_relative "hidden_row_tables"
|
|
4
6
|
|
|
5
7
|
module FunCi
|
|
6
8
|
module Persistence
|
|
@@ -73,21 +75,45 @@ module FunCi
|
|
|
73
75
|
%w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
|
|
74
76
|
%w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
|
|
75
77
|
%w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
|
|
76
|
-
%w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER]
|
|
78
|
+
%w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER],
|
|
79
|
+
%w[job_runs claimed_at TEXT], %w[job_runs began_at TEXT]].freeze
|
|
77
80
|
|
|
78
|
-
TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE
|
|
81
|
+
TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE,
|
|
82
|
+
*HiddenRowTables::ALL].freeze
|
|
83
|
+
|
|
84
|
+
# Where a commit arrived on its branch, which the board and the canceller
|
|
85
|
+
# look up for every run they read (Persistence::Arrival); runs are never
|
|
86
|
+
# deleted, so without it each lookup scans them all.
|
|
87
|
+
ARRIVAL_INDEX = "CREATE INDEX IF NOT EXISTS pipeline_runs_arrival " \
|
|
88
|
+
"ON pipeline_runs (project_path, branch, commit_hash)"
|
|
79
89
|
|
|
80
90
|
def self.migrate!(db)
|
|
81
91
|
with_setup_lock(db.filename("main")) do
|
|
82
92
|
TABLES.each { |table| db.execute(table) }
|
|
83
93
|
ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
|
|
94
|
+
db.execute(ARRIVAL_INDEX)
|
|
84
95
|
db.execute(NAME_DETACHED)
|
|
96
|
+
number_failures_kept_unnumbered(db)
|
|
85
97
|
end
|
|
86
98
|
end
|
|
87
99
|
|
|
88
100
|
# Runs started on a detached HEAD before 2.0.1 were kept with no branch name.
|
|
89
101
|
NAME_DETACHED = "UPDATE pipeline_runs SET branch = 'detached' WHERE branch = ''"
|
|
90
102
|
|
|
103
|
+
# In 2.4.0 and earlier, a slow stage whose process died was kept failed
|
|
104
|
+
# with no finish order, which a verdict needs to tell which failure came
|
|
105
|
+
# first.
|
|
106
|
+
FAILURES_KEPT_UNNUMBERED = "SELECT id FROM stage_jobs WHERE finished_order IS NULL " \
|
|
107
|
+
"AND status IN ('failed', 'timed_out') ORDER BY completed_at, id"
|
|
108
|
+
|
|
109
|
+
# Numbers each after the rest of its run, in the order they ended.
|
|
110
|
+
def self.number_failures_kept_unnumbered(db)
|
|
111
|
+
db.execute(FAILURES_KEPT_UNNUMBERED).each do |(id)|
|
|
112
|
+
db.execute("UPDATE stage_jobs SET finished_order = #{StageJob::NEXT_IN_RUN} WHERE id = ?", [id])
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
private_class_method :number_failures_kept_unnumbered
|
|
116
|
+
|
|
91
117
|
def self.with_setup_lock(db_path)
|
|
92
118
|
File.open("#{db_path}.setup-lock", File::RDWR | File::CREAT, 0o644) do |lock|
|
|
93
119
|
lock.flock(File::LOCK_EX)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Persistence
|
|
5
|
+
# The tables where the console keeps the rows the user hid, per machine
|
|
6
|
+
# (architecture.md, Daily and weekly jobs): JobVisibility's and BranchVisibility's.
|
|
7
|
+
module HiddenRowTables
|
|
8
|
+
JOBS = <<~SQL
|
|
9
|
+
CREATE TABLE IF NOT EXISTS hidden_jobs (
|
|
10
|
+
project_path TEXT, job TEXT, kind TEXT, hidden_at TEXT,
|
|
11
|
+
PRIMARY KEY (project_path, job)
|
|
12
|
+
)
|
|
13
|
+
SQL
|
|
14
|
+
|
|
15
|
+
BRANCHES = <<~SQL
|
|
16
|
+
CREATE TABLE IF NOT EXISTS hidden_branches (
|
|
17
|
+
project_path TEXT, branch TEXT, kind TEXT, hidden_at TEXT, PRIMARY KEY (project_path, branch)
|
|
18
|
+
)
|
|
19
|
+
SQL
|
|
20
|
+
|
|
21
|
+
ALL = [JOBS, BRANCHES].freeze
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -11,7 +11,7 @@ module FunCi
|
|
|
11
11
|
# and weekly jobs), apart from its pipeline runs.
|
|
12
12
|
class JobRuns
|
|
13
13
|
COLUMNS = %i[id project_path job cadence commit_hash branch status started_at completed_at lock_file pid
|
|
14
|
-
group_pid exit_status signal output_tail evidence budget].freeze
|
|
14
|
+
group_pid exit_status signal output_tail evidence budget claimed_at].freeze
|
|
15
15
|
SELECT = "SELECT #{COLUMNS.join(", ")} FROM job_runs".freeze
|
|
16
16
|
KEPT = 10
|
|
17
17
|
|
|
@@ -24,12 +24,14 @@ module FunCi
|
|
|
24
24
|
# and answers its id, when the job is due at `now`; nil otherwise. The
|
|
25
25
|
# job's latest run is read and the new one written in one transaction,
|
|
26
26
|
# so of two claims at once only one starts it. A claim forgets the
|
|
27
|
-
# job's runs beyond its newest 10.
|
|
28
|
-
|
|
27
|
+
# job's runs beyond its newest 10. A block is given the id inside that
|
|
28
|
+
# transaction, so what it records of the run (that it waits its turn)
|
|
29
|
+
# is there the moment anyone else can read the run.
|
|
30
|
+
def claim(job, commit:, lock_file:, now:, &then_record)
|
|
29
31
|
@db.transaction(:immediate) do
|
|
30
32
|
next nil unless Jobs::Due.new(latest(job.name), job.period, now: now).now?
|
|
31
33
|
|
|
32
|
-
insert(job, commit, lock_file, now).tap {
|
|
34
|
+
insert(job, commit, lock_file, now).tap { |id| claimed(job.name, id, then_record) }
|
|
33
35
|
end
|
|
34
36
|
end
|
|
35
37
|
|
|
@@ -53,13 +55,21 @@ module FunCi
|
|
|
53
55
|
|
|
54
56
|
def insert(job, commit, lock_file, now)
|
|
55
57
|
@db.execute("INSERT INTO job_runs (project_path, job, cadence, commit_hash, branch, status, started_at, " \
|
|
56
|
-
"lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)",
|
|
57
|
-
[@project, job.name, job.cadence, commit[:sha], commit[:branch], stamp(now), lock_file])
|
|
58
|
+
"claimed_at, began_at, lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, ?)",
|
|
59
|
+
[@project, job.name, job.cadence, commit[:sha], commit[:branch], *[stamp(now)] * 3, lock_file])
|
|
58
60
|
@db.last_insert_row_id
|
|
59
61
|
end
|
|
60
62
|
|
|
63
|
+
def claimed(name, id, then_record)
|
|
64
|
+
forget_beyond(name)
|
|
65
|
+
then_record&.call(id)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# A run began once it runs: began_at says when, and is none while it waits its turn.
|
|
61
69
|
def update(id, status, at, was)
|
|
62
|
-
|
|
70
|
+
began = stamp(at) if status == "running"
|
|
71
|
+
@db.execute("UPDATE job_runs SET status = ?, started_at = ?, began_at = ? WHERE id = ? AND #{was}",
|
|
72
|
+
[status, stamp(at), began, id])
|
|
63
73
|
@db.changes
|
|
64
74
|
end
|
|
65
75
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Persistence
|
|
7
|
+
# Which daily and weekly jobs the console keeps out of view, and how
|
|
8
|
+
# (design.md, The console): `ignored`, until it is shown again, or
|
|
9
|
+
# `snoozed`, until the job runs again. It says nothing of whether a job
|
|
10
|
+
# runs, and the agent commands don't read it.
|
|
11
|
+
class JobVisibility
|
|
12
|
+
# When a run began, if it did: not while it waits its turn, nor when
|
|
13
|
+
# cancelled before it (design.md, Daily and weekly jobs), and once it
|
|
14
|
+
# did, whatever became of it. fun-ci 2.4.0 and earlier kept no
|
|
15
|
+
# `began_at`, so of their runs those that ran or are running began when they started.
|
|
16
|
+
BEGAN = "COALESCE(run.began_at, CASE WHEN run.status IN ('running', 'completed', 'failed', 'timed_out') " \
|
|
17
|
+
"THEN run.started_at END)"
|
|
18
|
+
HIDDEN = <<~SQL.freeze
|
|
19
|
+
SELECT job, kind FROM hidden_jobs AS hidden WHERE project_path = ? AND (kind = 'ignored' OR NOT EXISTS (
|
|
20
|
+
SELECT 1 FROM job_runs AS run WHERE run.project_path = hidden.project_path AND run.job = hidden.job
|
|
21
|
+
AND #{BEGAN} > hidden.hidden_at))
|
|
22
|
+
SQL
|
|
23
|
+
|
|
24
|
+
def initialize(db)
|
|
25
|
+
@db = db
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Keeps the project's job `name` out of view from `at`, as `kind`:
|
|
29
|
+
# `snoozed` or `ignored`. The latest of two decides.
|
|
30
|
+
def hide(project, name, kind, at:)
|
|
31
|
+
@db.execute("INSERT OR REPLACE INTO hidden_jobs (project_path, job, kind, hidden_at) VALUES (?, ?, ?, ?)",
|
|
32
|
+
[project, name, kind, at.utc.iso8601(3)])
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def show(project, name)
|
|
36
|
+
@db.execute("DELETE FROM hidden_jobs WHERE project_path = ? AND job = ?", [project, name])
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# { name => kind } of the project's jobs out of view now.
|
|
40
|
+
def hidden(project) = @db.execute(HIDDEN, [project]).to_h
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -8,6 +8,9 @@ module FunCi
|
|
|
8
8
|
class NullRecorder
|
|
9
9
|
def create_run(**); end
|
|
10
10
|
def start_stage(_stage, **); end
|
|
11
|
+
def schedule_stage(_stage, **); end
|
|
12
|
+
def started_in_turn?(_job_id) = true
|
|
13
|
+
def arrival_place = 0
|
|
11
14
|
def end_stage(_job_id, _status); end
|
|
12
15
|
def keep_exit(_job_id, _exit_status, _signal); end
|
|
13
16
|
def keep_evidence(_job_id, _document); end
|
|
@@ -71,16 +71,31 @@ module FunCi
|
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
def start_stage(stage, budget: nil)
|
|
74
|
+
schedule_stage(stage, budget: budget).tap { |job_id| stage_started(job_id) if job_id }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# A stage that waits before it starts, as a slow suite does for its turn.
|
|
78
|
+
def schedule_stage(stage, budget: nil)
|
|
74
79
|
return nil unless @pipeline_run_id
|
|
75
80
|
|
|
76
81
|
tolerating do
|
|
77
82
|
ensure_running
|
|
78
|
-
|
|
79
|
-
StageJob.update_status(@db, job_id, "running")
|
|
80
|
-
job_id
|
|
83
|
+
StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
|
|
81
84
|
end
|
|
82
85
|
end
|
|
83
86
|
|
|
87
|
+
# Records a slow suite started once its turn comes; false when its run
|
|
88
|
+
# was cancelled while it waited. One whose stage could not be recorded,
|
|
89
|
+
# or whose start the database refused, starts.
|
|
90
|
+
def started_in_turn?(job_id)
|
|
91
|
+
return true unless job_id
|
|
92
|
+
|
|
93
|
+
tolerating { StageJob.started_in_turn?(@db, job_id) } != false
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Where this run's commit arrived on its branch (Persistence::Arrival); 0 when not known.
|
|
97
|
+
def arrival_place = (@pipeline_run_id && tolerating { PipelineRun.arrival_place(@db, @pipeline_run_id) }) || 0
|
|
98
|
+
|
|
84
99
|
# Keeps a failed stage's evidence, and the output's tail where an older
|
|
85
100
|
# fun-ci sharing the database reads it.
|
|
86
101
|
def keep_evidence(job_id, document)
|
|
@@ -113,6 +128,10 @@ module FunCi
|
|
|
113
128
|
|
|
114
129
|
private
|
|
115
130
|
|
|
131
|
+
def stage_started(job_id)
|
|
132
|
+
tolerating { StageJob.update_status(@db, job_id, "running") }
|
|
133
|
+
end
|
|
134
|
+
|
|
116
135
|
def ensure_running
|
|
117
136
|
run = PipelineRun.find(@db, @pipeline_run_id)
|
|
118
137
|
return unless run && run[:status] == "scheduled"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "time"
|
|
4
|
+
require_relative "arrival"
|
|
4
5
|
|
|
5
6
|
module FunCi
|
|
6
7
|
module Persistence
|
|
@@ -56,6 +57,11 @@ module FunCi
|
|
|
56
57
|
db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
|
|
57
58
|
end
|
|
58
59
|
|
|
60
|
+
# The id of the first run of this run's commit on its branch: where the commit arrived.
|
|
61
|
+
def self.arrival_place(db, id)
|
|
62
|
+
db.get_first_value("SELECT #{Arrival::FIRST_RUN} FROM pipeline_runs WHERE id = ?", [id])
|
|
63
|
+
end
|
|
64
|
+
|
|
59
65
|
def self.update_status(db, id, new_status, at: Time.now)
|
|
60
66
|
db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, at.utc.iso8601, id])
|
|
61
67
|
end
|
|
@@ -69,7 +75,10 @@ module FunCi
|
|
|
69
75
|
"WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
|
|
70
76
|
|
|
71
77
|
# Each branch's newest run.
|
|
72
|
-
|
|
78
|
+
# Each branch's head: the newest run of the commit that arrived on it last (Arrival).
|
|
79
|
+
HEADS = "SELECT MAX(id) FROM pipeline_runs WHERE #{Arrival::FIRST_RUN} IN " \
|
|
80
|
+
"(SELECT MAX(#{Arrival::FIRST_RUN}) FROM pipeline_runs GROUP BY project_path, branch) " \
|
|
81
|
+
"GROUP BY project_path, branch".freeze
|
|
73
82
|
|
|
74
83
|
# The newest run of each of `limit` branches, of the `among` whose newest
|
|
75
84
|
# run changed most recently: those that most need you, then those run
|
|
@@ -79,9 +88,11 @@ module FunCi
|
|
|
79
88
|
"ORDER BY updated_at DESC, id DESC LIMIT ?) ORDER BY #{NEEDING}, id DESC LIMIT ?", among, limit)
|
|
80
89
|
end
|
|
81
90
|
|
|
82
|
-
#
|
|
83
|
-
|
|
84
|
-
|
|
91
|
+
# The runs of `head`'s branch up to it, newest first: not a run of an
|
|
92
|
+
# older commit run again since.
|
|
93
|
+
def self.leading_to(db, head, limit:)
|
|
94
|
+
query(db, "WHERE project_path IS ? AND branch = ? AND id <= ? ORDER BY id DESC LIMIT ?",
|
|
95
|
+
head[:project_path], head[:branch], head[:id], limit)
|
|
85
96
|
end
|
|
86
97
|
|
|
87
98
|
def self.query(db, clause, *params)
|
|
@@ -22,6 +22,19 @@ module FunCi
|
|
|
22
22
|
db.last_insert_row_id
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
+
# Records a slow suite waiting its branch's turn running, unless it or its
|
|
26
|
+
# run was recorded cancelled meanwhile; whether it was. One statement, so
|
|
27
|
+
# a cancel recorded before it refuses it and one after finds it running
|
|
28
|
+
# (acceptance-tests.md, AT-9.19).
|
|
29
|
+
def self.started_in_turn?(db, id)
|
|
30
|
+
db.execute("UPDATE stage_jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'scheduled' " \
|
|
31
|
+
"AND NOT EXISTS (#{RUN_CANCELLED})", [Time.now.utc.iso8601(3), id])
|
|
32
|
+
db.changes == 1
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
RUN_CANCELLED = "SELECT 1 FROM pipeline_runs WHERE pipeline_runs.id = stage_jobs.pipeline_run_id " \
|
|
36
|
+
"AND pipeline_runs.status = 'cancelled'"
|
|
37
|
+
|
|
25
38
|
# The stage script's process, which leads a process group of its own.
|
|
26
39
|
def self.store_pid(db, id, pid)
|
|
27
40
|
db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "time"
|
|
5
5
|
require_relative "../trunk/check"
|
|
6
|
+
require_relative "pipeline_run"
|
|
6
7
|
|
|
7
8
|
module FunCi
|
|
8
9
|
module Persistence
|
|
@@ -12,7 +13,7 @@ module FunCi
|
|
|
12
13
|
TIP = %i[trunk_remote trunk_branch trunk_sha trunk_seen_at].freeze
|
|
13
14
|
MERGE = %i[outcome ahead behind files reason].freeze
|
|
14
15
|
COLUMNS = [:commit_hash, *TIP, *MERGE].freeze
|
|
15
|
-
SELECT = "SELECT #{COLUMNS.join(", ")} FROM trunk_checks".freeze
|
|
16
|
+
SELECT = "SELECT #{COLUMNS.join(", ")}, checked_at FROM trunk_checks".freeze
|
|
16
17
|
|
|
17
18
|
def initialize(db, project)
|
|
18
19
|
@db = db
|
|
@@ -36,14 +37,14 @@ module FunCi
|
|
|
36
37
|
found = @db.execute("#{SELECT} WHERE project_path = ? AND commit_hash = ? " \
|
|
37
38
|
"ORDER BY COALESCE(trunk_seen_at, checked_at) DESC, id DESC LIMIT 1", [@project, sha]).first
|
|
38
39
|
found && Trunk::Check.new(commit: found.first, tip: tip(found[1, TIP.size]),
|
|
39
|
-
merge: merge(found.
|
|
40
|
+
merge: merge(found[1 + TIP.size, MERGE.size]), checked_at: Time.parse(found.last))
|
|
40
41
|
end
|
|
41
42
|
|
|
42
|
-
# { commit => trunk SHA } for each branch's
|
|
43
|
-
# whose commit has been checked against a trunk tip.
|
|
43
|
+
# { commit => trunk SHA } for each branch's head (PipelineRun::HEADS), other
|
|
44
|
+
# than `except`, whose commit has been checked against a trunk tip.
|
|
44
45
|
def branch_heads(except:)
|
|
45
|
-
heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (
|
|
46
|
-
"
|
|
46
|
+
heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (#{PipelineRun::HEADS}) " \
|
|
47
|
+
"AND project_path = ? AND commit_hash != ?", [@project, except]).flatten
|
|
47
48
|
heads.to_h { |sha| [sha, latest(sha)&.tip&.sha] }.compact
|
|
48
49
|
end
|
|
49
50
|
|
|
@@ -29,13 +29,14 @@ module FunCi
|
|
|
29
29
|
# fun-ci's own, the window what it prints is written to, what to do
|
|
30
30
|
# with its pid when it runs over budget, before it is killed, the
|
|
31
31
|
# open files it holds as long as it runs, such as a lock, and what it
|
|
32
|
-
# is run under (a Priorities prefix, "" for fun-ci's own priority)
|
|
33
|
-
|
|
32
|
+
# is run under (a Priorities prefix, "" for fun-ci's own priority), and
|
|
33
|
+
# how long its output may drain once it is killed.
|
|
34
|
+
Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority, :drain)
|
|
34
35
|
|
|
35
36
|
class Launch
|
|
36
37
|
def initialize(**given)
|
|
37
38
|
super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
|
|
38
|
-
**given)
|
|
39
|
+
drain: DRAIN_SECONDS, **given)
|
|
39
40
|
end
|
|
40
41
|
end
|
|
41
42
|
|
|
@@ -100,12 +101,12 @@ module FunCi
|
|
|
100
101
|
def over_budget(pid, reading, launch)
|
|
101
102
|
launch.before_kill&.call(pid)
|
|
102
103
|
kill_process_group(pid)
|
|
103
|
-
[drained(reading, launch
|
|
104
|
+
[drained(reading, launch), nil, true]
|
|
104
105
|
end
|
|
105
106
|
|
|
106
|
-
def drained(reading,
|
|
107
|
-
reading.join(
|
|
108
|
-
text(
|
|
107
|
+
def drained(reading, launch)
|
|
108
|
+
reading.join(launch.drain)
|
|
109
|
+
text(launch.output)
|
|
109
110
|
end
|
|
110
111
|
|
|
111
112
|
def text(window) = window.text.force_encoding(Encoding.default_external)
|
|
@@ -81,9 +81,7 @@ module FunCi
|
|
|
81
81
|
# records a stage's group before it lets the stage start, so once those
|
|
82
82
|
# processes are dead, looking again finds every stage they started.
|
|
83
83
|
def stop_late_stages(db, run)
|
|
84
|
-
Persistence::ActiveRuns.
|
|
85
|
-
(now.stage_groups - run.stage_groups).each { |group| kill(-group) }
|
|
86
|
-
end
|
|
84
|
+
(Persistence::ActiveRuns.groups(db, run.id) - run.stage_groups).each { |group| kill(-group) }
|
|
87
85
|
end
|
|
88
86
|
|
|
89
87
|
# The groups recorded since `before` was read.
|
data/lib/fun_ci/pipeline/slot.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "waiting_slots"
|
|
4
|
+
|
|
3
5
|
module FunCi
|
|
4
6
|
module Pipeline
|
|
5
7
|
# A worktree a pipeline has to itself, held through an flock on its lock
|
|
@@ -26,6 +28,16 @@ module FunCi
|
|
|
26
28
|
@lock.close if @holders.zero? && !@lock.closed?
|
|
27
29
|
end
|
|
28
30
|
|
|
31
|
+
# Marks the slot as held only by a slow suite waiting its turn, until
|
|
32
|
+
# the mark is closed (WaitingSlots). A slot with nothing to lock has
|
|
33
|
+
# nothing to mark.
|
|
34
|
+
def waiting = lock_file ? WaitingSlots.mark(path) : Unmarked.new
|
|
35
|
+
|
|
36
|
+
# The mark of a slot with nothing to lock: closing it does nothing.
|
|
37
|
+
class Unmarked
|
|
38
|
+
def close; end
|
|
39
|
+
end
|
|
40
|
+
|
|
29
41
|
# Whether its worktree was just made, so has no caches yet.
|
|
30
42
|
def cold? = false
|
|
31
43
|
|
|
@@ -5,6 +5,7 @@ require_relative "background_fork"
|
|
|
5
5
|
require_relative "stage_runner"
|
|
6
6
|
require_relative "stage_execution"
|
|
7
7
|
require_relative "progress_reporter"
|
|
8
|
+
require_relative "turn_queue"
|
|
8
9
|
|
|
9
10
|
module FunCi
|
|
10
11
|
module Pipeline
|
|
@@ -71,26 +72,40 @@ module FunCi
|
|
|
71
72
|
end
|
|
72
73
|
|
|
73
74
|
def launch_slow_suite(config)
|
|
74
|
-
job_id = recorder.
|
|
75
|
+
job_id = recorder.schedule_stage("slow", budget: @seams.budgets["slow"])
|
|
75
76
|
launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
|
|
76
77
|
executor: slow_suite(config, @slot.share))
|
|
77
78
|
end
|
|
78
79
|
|
|
79
80
|
# Called with the recorder and job to record the slow suite with, in
|
|
80
|
-
# whichever process runs it
|
|
81
|
+
# whichever process runs it, once no older slow suite of the branch runs
|
|
82
|
+
# (acceptance-tests.md, AT-9.19); the slot is let go once it is recorded.
|
|
81
83
|
def slow_suite(config, slot)
|
|
82
84
|
execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
|
|
83
85
|
launching: { priority: @seams.priorities.slow })
|
|
84
86
|
command = config.stage_command("slow", @commit.sha)
|
|
85
|
-
->(recorder, job_id) {
|
|
87
|
+
->(recorder, job_id) { in_turn(slot, recorder, job_id) { execution.run("slow", command, recorder, job_id) } }
|
|
86
88
|
end
|
|
87
89
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
# Holding the slot until the slow suite is recorded, whether or not its
|
|
91
|
+
# turn came, marked waiting until it does (WaitingSlots); a run
|
|
92
|
+
# cancelled while it waited never starts it.
|
|
93
|
+
def in_turn(slot, recorder, job_id, &)
|
|
94
|
+
waiting = slot.waiting
|
|
95
|
+
take_turn(waiting, recorder, job_id, &)
|
|
90
96
|
ensure
|
|
97
|
+
waiting&.close
|
|
91
98
|
slot.release
|
|
92
99
|
end
|
|
93
100
|
|
|
101
|
+
def take_turn(waiting, recorder, job_id)
|
|
102
|
+
ticket = TurnTicket.new(place: recorder.arrival_place, id: job_id.to_i)
|
|
103
|
+
@seams.slow_turns.take(@commit.branch, ticket) do
|
|
104
|
+
waiting.close
|
|
105
|
+
yield if recorder.started_in_turn?(job_id)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
94
109
|
def launch(**)
|
|
95
110
|
return @seams.background_launcher.call(**) if @seams.background_launcher
|
|
96
111
|
|