fun_ci 2.3.0-aarch64-linux → 2.5.0-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +143 -0
- data/README.md +20 -10
- data/lib/fun_ci/agent/events.rb +15 -7
- data/lib/fun_ci/agent/job_events.rb +12 -5
- data/lib/fun_ci/agent/onward.rb +21 -0
- data/lib/fun_ci/agent/output.rb +7 -0
- data/lib/fun_ci/agent/reports.rb +3 -0
- data/lib/fun_ci/agent/snapshots.rb +25 -7
- data/lib/fun_ci/agent/status_text.rb +9 -5
- data/lib/fun_ci/agent/wait_command.rb +37 -5
- data/lib/fun_ci/cli.rb +2 -0
- data/lib/fun_ci/cli_command_help.rb +51 -0
- data/lib/fun_ci/cli_help.rb +9 -9
- data/lib/fun_ci/console/board_data.rb +50 -10
- data/lib/fun_ci/console/branch_page.rb +9 -0
- data/lib/fun_ci/console/console_state.rb +22 -8
- data/lib/fun_ci/console/cursor.rb +22 -0
- data/lib/fun_ci/console/cursor_rows.rb +37 -0
- data/lib/fun_ci/console/job_message.rb +5 -4
- data/lib/fun_ci/console/job_order.rb +5 -3
- data/lib/fun_ci/console/job_row.rb +17 -0
- data/lib/fun_ci/console/job_rows.rb +14 -3
- data/lib/fun_ci/console/key_handler.rb +88 -51
- data/lib/fun_ci/console/question.rb +63 -0
- data/lib/fun_ci/console/run_message.rb +3 -3
- data/lib/fun_ci/console/trunk_marks.rb +3 -1
- data/lib/fun_ci/console/view.rb +20 -6
- data/lib/fun_ci/evidence/about_stage.rb +1 -1
- data/lib/fun_ci/evidence/collector.rb +6 -3
- data/lib/fun_ci/evidence/outcome.rb +6 -3
- data/lib/fun_ci/jobs/job_run.rb +10 -9
- data/lib/fun_ci/jobs/locks.rb +1 -1
- data/lib/fun_ci/persistence/active_jobs.rb +2 -11
- data/lib/fun_ci/persistence/active_runs.rb +48 -16
- data/lib/fun_ci/persistence/arrival.rb +18 -0
- data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
- data/lib/fun_ci/persistence/database.rb +28 -2
- data/lib/fun_ci/persistence/died.rb +21 -0
- data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
- data/lib/fun_ci/persistence/job_recorder.rb +1 -0
- data/lib/fun_ci/persistence/job_runs.rb +17 -7
- data/lib/fun_ci/persistence/job_visibility.rb +43 -0
- data/lib/fun_ci/persistence/null_recorder.rb +4 -0
- data/lib/fun_ci/persistence/pipeline_recorder.rb +23 -3
- data/lib/fun_ci/persistence/pipeline_run.rb +25 -11
- data/lib/fun_ci/persistence/stage_job.rb +25 -0
- data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
- data/lib/fun_ci/pipeline/group_kill.rb +38 -0
- data/lib/fun_ci/pipeline/pipeline_forker.rb +3 -1
- data/lib/fun_ci/pipeline/process_runner.rb +14 -8
- data/lib/fun_ci/pipeline/process_tree.rb +37 -0
- data/lib/fun_ci/pipeline/progress_reporter.rb +2 -1
- data/lib/fun_ci/pipeline/run_canceller.rb +16 -6
- data/lib/fun_ci/pipeline/slot.rb +12 -0
- data/lib/fun_ci/pipeline/slot_run.rb +20 -5
- data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
- data/lib/fun_ci/pipeline/stage_end.rb +2 -1
- data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
- data/lib/fun_ci/pipeline/trigger.rb +9 -1
- data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
- data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
- data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
- data/lib/fun_ci/pipeline/workspaces.rb +2 -1
- data/lib/fun_ci/pipeline/worktree_pool.rb +27 -7
- data/lib/fun_ci/pipeline/worktree_prune.rb +9 -4
- data/lib/fun_ci/pipeline/worktrees.rb +41 -7
- data/lib/fun_ci/setup/commands.rb +16 -7
- data/lib/fun_ci/setup/eslint_detection.rb +38 -0
- data/lib/fun_ci/setup/init_options.rb +34 -0
- data/lib/fun_ci/setup/installer.rb +41 -7
- data/lib/fun_ci/setup/placeholders.rb +30 -0
- data/lib/fun_ci/setup/project_config.rb +1 -0
- data/lib/fun_ci/setup/settings.rb +21 -1
- data/lib/fun_ci/setup/setup_checker.rb +5 -3
- data/lib/fun_ci/setup/stage_templates/javascript.rb +14 -4
- data/lib/fun_ci/setup/stage_templates/placeholders.rb +21 -0
- data/lib/fun_ci/setup/stage_templates.rb +6 -4
- data/lib/fun_ci/setup/template_writer.rb +3 -5
- data/lib/fun_ci/trunk/check.rb +7 -2
- data/lib/fun_ci/version.rb +1 -1
- data/libexec/fun-ci-renderer +0 -0
- metadata +33 -10
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "cursor_rows"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Console
|
|
7
|
+
# What the footer asks about the row under the cursor before a key acts
|
|
8
|
+
# on it (design.md, The console). Each says what it `asks`, does what a
|
|
9
|
+
# key it is given answers (`take`), and says whether that key ends it.
|
|
10
|
+
module Question
|
|
11
|
+
# What each question holds, the board it acts on and the row it asks
|
|
12
|
+
# about, and which keys end it: its class's ENDS.
|
|
13
|
+
module Asked
|
|
14
|
+
# kind: :run or :job; row: the branch's run, or the job's row.
|
|
15
|
+
def initialize(board_data, kind, row)
|
|
16
|
+
@board_data = board_data
|
|
17
|
+
@kind = kind
|
|
18
|
+
@row = row
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def ended_by?(key) = self.class::ENDS.include?(key)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Whether to cancel a running run or job, whose processes get killed: `y` does.
|
|
25
|
+
class Cancel
|
|
26
|
+
include Asked
|
|
27
|
+
|
|
28
|
+
ENDS = ["y", "n", :escape].freeze
|
|
29
|
+
|
|
30
|
+
def asks = :cancel
|
|
31
|
+
|
|
32
|
+
def take(key) = key == "y" && carry_out
|
|
33
|
+
|
|
34
|
+
def carry_out = @kind == :job ? @board_data.cancel_job(@row.run[:id]) : @board_data.cancel_run(@row[:id])
|
|
35
|
+
|
|
36
|
+
# Whether `place`, [kind, row] under the cursor now, is still the run asked about, running.
|
|
37
|
+
def about?(place) = place.first == @kind && Cancel.target(*place) == Cancel.target(@kind, @row)
|
|
38
|
+
|
|
39
|
+
# A run's id and status, or a job's latest run's id and its state.
|
|
40
|
+
def self.target(kind, row) = kind == :job ? [row.run&.dig(:id), row.state] : row.values_at(:id, :status)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# How to hide a branch or a job: `s` snoozes it, until its next run; `i` ignores it, until shown again.
|
|
44
|
+
class Hide
|
|
45
|
+
include Asked
|
|
46
|
+
|
|
47
|
+
KINDS = { "s" => "snoozed", "i" => "ignored" }.freeze
|
|
48
|
+
ENDS = [*KINDS.keys, "n", :escape].freeze
|
|
49
|
+
|
|
50
|
+
def asks = :hide
|
|
51
|
+
|
|
52
|
+
def take(key) = KINDS.key?(key) && hide(KINDS[key])
|
|
53
|
+
|
|
54
|
+
# Whether `place`, [kind, row] under the cursor now, is still the branch or job asked about.
|
|
55
|
+
def about?(place) = place.first == @kind && CursorRows.key(*place) == CursorRows.key(@kind, @row)
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def hide(how) = @kind == :job ? @board_data.hide_job(@row, how) : @board_data.hide_run(@row, how)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -16,9 +16,9 @@ module FunCi
|
|
|
16
16
|
stages: run[:stages].map { |stage| stage(stage) }, **marks(run) }.compact
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
# What the row marks beyond the run itself: a conflict with the trunk,
|
|
20
|
-
# how many cancelled runs it stands for.
|
|
21
|
-
def self.marks(run) = { trunk: conflict(run[:trunk]), folded: run[:folded] }
|
|
19
|
+
# What the row marks beyond the run itself: a conflict with the trunk,
|
|
20
|
+
# how many cancelled runs it stands for, and how the user hid its branch.
|
|
21
|
+
def self.marks(run) = { trunk: conflict(run[:trunk]), folded: run[:folded], hidden: run[:hidden] }
|
|
22
22
|
|
|
23
23
|
# The branch's standing against the trunk, sent only where the row shows it: a conflict.
|
|
24
24
|
def self.conflict(trunk) = trunk && trunk[:branch_state] == "conflicts" ? trunk : nil
|
|
@@ -44,10 +44,12 @@ module FunCi
|
|
|
44
44
|
|
|
45
45
|
def stale_since(project, now)
|
|
46
46
|
last = Persistence::TrunkFetches.new(@db, project).last
|
|
47
|
-
return nil unless last && (last.error ||
|
|
47
|
+
return nil unless last && (last.error || old?(last.fetched_at, now))
|
|
48
48
|
|
|
49
49
|
{ project: project, since: last.fetched_at&.to_i }
|
|
50
50
|
end
|
|
51
|
+
|
|
52
|
+
def old?(fetched_at, now) = !fetched_at.nil? && now - fetched_at > Trunk::Shown::FRESH_FOR
|
|
51
53
|
end
|
|
52
54
|
end
|
|
53
55
|
end
|
data/lib/fun_ci/console/view.rb
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module FunCi
|
|
4
4
|
module Console
|
|
5
|
-
# What the user sees of the rows and does with them: KeyHandler's cursor
|
|
6
|
-
# and
|
|
5
|
+
# What the user sees of the rows and does with them: KeyHandler's cursor,
|
|
6
|
+
# what it asks and which job rows it keeps in view, and one page of rows, one per branch, as many as
|
|
7
7
|
# the terminal fits, scrolled so the row under the cursor is on it
|
|
8
8
|
# (renderer-protocol.md, `board`).
|
|
9
9
|
class View
|
|
@@ -19,25 +19,39 @@ module FunCi
|
|
|
19
19
|
@page_size = 0
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
+
# The least a page holds: the renderer fits what it can of a run, and
|
|
23
|
+
# a page of none would read as no runs at all.
|
|
24
|
+
LEAST = 1
|
|
25
|
+
|
|
22
26
|
# The number of runs a terminal of `rows` fits.
|
|
23
27
|
def resize(rows)
|
|
24
|
-
@page_size = [rows - CHROME_ROWS,
|
|
28
|
+
@page_size = [rows - CHROME_ROWS, LEAST].max
|
|
25
29
|
end
|
|
26
30
|
|
|
27
31
|
# :quit when the key ends the session.
|
|
28
32
|
def press(key) = @key_handler.handle_key(key)
|
|
29
33
|
|
|
34
|
+
# The job rows in view: not the hidden ones, unless the user asked for them.
|
|
35
|
+
def shown(job_rows) = @key_handler.shown(job_rows)
|
|
36
|
+
|
|
37
|
+
# The branches' runs in view: not those of hidden branches, unless the user asked for them.
|
|
38
|
+
def shown_runs(runs) = @key_handler.shown_runs(runs)
|
|
39
|
+
|
|
30
40
|
# `more`: whether the store holds runs beyond `runs`. `jobs`: the job
|
|
31
|
-
# section's rows, below the page, which the cursor reaches after the runs.
|
|
41
|
+
# section's rows in view, below the page, which the cursor reaches after the runs.
|
|
32
42
|
def page(runs, more:, jobs: [])
|
|
33
|
-
|
|
43
|
+
@key_handler.follow(runs, jobs, from_the_first: true)
|
|
44
|
+
size = jobs.empty? ? @page_size : [@page_size - JOB_LINES, LEAST].max
|
|
34
45
|
first = first_shown(runs.size, size)
|
|
35
|
-
{ cursor: cursor_on_page(first, size), confirming: @key_handler.confirming?,
|
|
46
|
+
{ cursor: cursor_on_page(first, size), confirming: @key_handler.confirming?, **hiding,
|
|
36
47
|
has_more: more || first + size < runs.size, runs: runs[first, size] || [] }
|
|
37
48
|
end
|
|
38
49
|
|
|
39
50
|
private
|
|
40
51
|
|
|
52
|
+
# Only while the footer asks how to hide a branch or a job, as the protocol lets a board leave it out.
|
|
53
|
+
def hiding = @key_handler.asking == :hide ? { hiding: true } : {}
|
|
54
|
+
|
|
41
55
|
# The last page of runs while the cursor is on a job.
|
|
42
56
|
def first_shown(count, size)
|
|
43
57
|
cursor = @key_handler.cursor_index
|
|
@@ -10,7 +10,7 @@ module FunCi
|
|
|
10
10
|
# command to read.
|
|
11
11
|
module AboutStage
|
|
12
12
|
def self.of(sources, outcome, output)
|
|
13
|
-
About.new(**outcome.to_h.except(:overrun), **started(sources), **where(sources, output),
|
|
13
|
+
About.new(**outcome.to_h.except(:overrun, :elsewhere), **started(sources), **where(sources, output),
|
|
14
14
|
stage: sources.stage, budget: sources.budget, commit: sources.commit)
|
|
15
15
|
end
|
|
16
16
|
|
|
@@ -37,7 +37,7 @@ module FunCi
|
|
|
37
37
|
def collect(output, outcome = FAILED)
|
|
38
38
|
result = extracted(output, outcome)
|
|
39
39
|
overrun = outcome.overrun || NOTHING
|
|
40
|
-
parts = [["fun-ci", own_facts(outcome
|
|
40
|
+
parts = [["fun-ci", own_facts(outcome)], *overrun.parts, *result.parts, ["output-tail", tail(output)]]
|
|
41
41
|
Caps.new.apply(masking.document(assemble(parts, [overrun, result])))
|
|
42
42
|
end
|
|
43
43
|
|
|
@@ -76,10 +76,13 @@ module FunCi
|
|
|
76
76
|
|
|
77
77
|
def on_of(raw) = raw.is_a?(Hash) ? raw["on"] : nil
|
|
78
78
|
|
|
79
|
-
def own_facts(
|
|
80
|
-
|
|
79
|
+
def own_facts(outcome)
|
|
80
|
+
named = { "alongside" => outcome.alongside, "other runs alongside" => outcome.elsewhere }
|
|
81
|
+
Findings.new(facts: named.reject { |_, stages| stages.empty? }.map { |name, stages| fact(name, stages) })
|
|
81
82
|
end
|
|
82
83
|
|
|
84
|
+
def fact(name, stages) = { name: name, value: stages.join(", ") }
|
|
85
|
+
|
|
83
86
|
def tail(output)
|
|
84
87
|
text = Persistence::OutputTail.of(output, mask: masking.method(:mask))
|
|
85
88
|
Findings.new(excerpts: [{ title: Document::TAIL_TITLE, location: "output", lines: text.lines(chomp: true) }])
|
|
@@ -4,12 +4,15 @@ module FunCi
|
|
|
4
4
|
module Evidence
|
|
5
5
|
# How a failed stage ended, known once it has: its state in the words
|
|
6
6
|
# `status --json` uses, its exit status or signal, and the other stages
|
|
7
|
-
# that shared its slot while it ran,
|
|
7
|
+
# that shared its slot while it ran, the stages of the project's other runs
|
|
8
|
+
# that ran meanwhile (elsewhere), and for an overrun, what was found
|
|
8
9
|
# before the kill (an Extraction::Result).
|
|
9
|
-
Outcome = Data.define(:state, :exit_status, :signal, :alongside, :overrun)
|
|
10
|
+
Outcome = Data.define(:state, :exit_status, :signal, :alongside, :elsewhere, :overrun)
|
|
10
11
|
|
|
11
12
|
class Outcome
|
|
12
|
-
def initialize(**given)
|
|
13
|
+
def initialize(**given)
|
|
14
|
+
super(exit_status: nil, signal: nil, alongside: [], elsewhere: [], overrun: nil, **given)
|
|
15
|
+
end
|
|
13
16
|
end
|
|
14
17
|
end
|
|
15
18
|
end
|
data/lib/fun_ci/jobs/job_run.rb
CHANGED
|
@@ -44,22 +44,23 @@ module FunCi
|
|
|
44
44
|
|
|
45
45
|
def claimed(slot, at)
|
|
46
46
|
runs.died(@job.name)
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
now = @site.clock.call
|
|
48
|
+
waits = at > now
|
|
49
|
+
id = runs.claim(@job, commit: @commit.to_h, lock_file: slot.lock_file, now: now) do |run|
|
|
50
|
+
scheduled(run, at) if waits
|
|
51
|
+
end
|
|
52
|
+
id && (!waits || turn_came?(id, at - now) ? execute(slot, id) : id)
|
|
49
53
|
end
|
|
50
54
|
|
|
51
|
-
# Waits
|
|
55
|
+
# Waits `seconds` for the run's turn, the lock held; false when it was
|
|
52
56
|
# cancelled meanwhile.
|
|
53
|
-
def turn_came?(id,
|
|
54
|
-
seconds = at - @site.clock.call
|
|
55
|
-
return true unless seconds.positive?
|
|
56
|
-
|
|
57
|
-
scheduled(id, at)
|
|
57
|
+
def turn_came?(id, seconds)
|
|
58
58
|
@site.wait.call(seconds)
|
|
59
59
|
runs.start_turn(id, @site.clock.call).positive?
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
-
#
|
|
62
|
+
# Scheduled from the moment it is claimed, and this process recorded, so
|
|
63
|
+
# a cancel stops it while it waits.
|
|
63
64
|
def scheduled(id, at)
|
|
64
65
|
recorder.started_by(id, Process.pid, budget: budget)
|
|
65
66
|
runs.wait_until(id, at)
|
data/lib/fun_ci/jobs/locks.rb
CHANGED
|
@@ -6,7 +6,7 @@ require_relative "../pipeline/slot"
|
|
|
6
6
|
module FunCi
|
|
7
7
|
module Jobs
|
|
8
8
|
# Each job's worktree and the lock beside it, under a project's
|
|
9
|
-
#
|
|
9
|
+
# Worktrees#jobs_root: `<name>` and `<name>.lock`. Whoever holds
|
|
10
10
|
# a job's lock runs it, so it runs once at a time, and a lock dies with
|
|
11
11
|
# the process that held it.
|
|
12
12
|
class Locks
|
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "time"
|
|
4
4
|
require_relative "active_runs"
|
|
5
|
-
|
|
6
|
-
require_relative "../evidence/document"
|
|
5
|
+
require_relative "died"
|
|
7
6
|
|
|
8
7
|
module FunCi
|
|
9
8
|
module Persistence
|
|
@@ -25,14 +24,6 @@ module FunCi
|
|
|
25
24
|
# [id, lock file] of each job run still running.
|
|
26
25
|
def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE #{ACTIVE}")
|
|
27
26
|
|
|
28
|
-
# What a run whose process died keeps: that it has no result, and why
|
|
29
|
-
# that may be. It keeps no end, since when it was found dead says
|
|
30
|
-
# nothing of how long it ran.
|
|
31
|
-
DIED_FACT = { name: "ended", extractor: "fun-ci",
|
|
32
|
-
value: "its process stopped before it said how the run ended (the machine slept or " \
|
|
33
|
-
"restarted, or something killed it), so this run has no result" }.freeze
|
|
34
|
-
DIED = Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [DIED_FACT])
|
|
35
|
-
|
|
36
27
|
def self.cancelled(db, id) = ended(db, id, "cancelled", ACTIVE)
|
|
37
28
|
|
|
38
29
|
# A run whose process is gone without recording how it ended: failed,
|
|
@@ -41,7 +32,7 @@ module FunCi
|
|
|
41
32
|
def self.died(db, id)
|
|
42
33
|
ended(db, id, "cancelled", "status = 'scheduled'")
|
|
43
34
|
db.execute("UPDATE job_runs SET status = 'failed', evidence = ? WHERE id = ? AND status = 'running'",
|
|
44
|
-
[
|
|
35
|
+
[Died.evidence("run"), id])
|
|
45
36
|
end
|
|
46
37
|
|
|
47
38
|
# The ids of a project's job's runs still running.
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require "time"
|
|
4
4
|
require_relative "pipeline_run"
|
|
5
5
|
require_relative "stage_job"
|
|
6
|
+
require_relative "died"
|
|
7
|
+
require_relative "arrival"
|
|
6
8
|
|
|
7
9
|
module FunCi
|
|
8
10
|
module Persistence
|
|
@@ -19,30 +21,49 @@ module FunCi
|
|
|
19
21
|
module ActiveRuns
|
|
20
22
|
ACTIVE = "status IN ('scheduled', 'running')"
|
|
21
23
|
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
# Records cancelled, and returns, the runs on the branch a new commit's
|
|
25
|
+
# run cancels: those of commits that arrived before it (acceptance-tests.md,
|
|
26
|
+
# AT-9.20), except one an agent has waited on since `waited_before` (an
|
|
27
|
+
# ISO 8601 UTC time; AT-9.12) and one whose slow suite is running (AT-9.19).
|
|
28
|
+
# Picked and recorded in one statement, before their processes are
|
|
29
|
+
# stopped, so a slow suite whose turn comes meanwhile never starts
|
|
30
|
+
# (StageJob.started_in_turn?).
|
|
31
|
+
def self.cancel_stale(db, branch, new_commit:, waited_before:)
|
|
32
|
+
values = [Time.now.utc.iso8601, branch.project, branch.name, branch.project, branch.name, new_commit,
|
|
33
|
+
waited_before]
|
|
34
|
+
rows = db.execute("UPDATE pipeline_runs SET status = 'cancelled', updated_at = ? WHERE #{STALE} " \
|
|
35
|
+
"RETURNING #{RUN_COLUMNS}", values)
|
|
36
|
+
rows.sort_by(&:first).map { |row| active_run(db, row) }
|
|
28
37
|
end
|
|
29
38
|
|
|
39
|
+
# A commit arrives on its branch with its first run; a commit with none yet arrives last.
|
|
40
|
+
ARRIVED_BEFORE = "#{Arrival::FIRST_RUN} < COALESCE((SELECT MIN(id) FROM pipeline_runs " \
|
|
41
|
+
"WHERE project_path IS ? AND branch = ? AND commit_hash = ?), #{Arrival::NOT_YET})".freeze
|
|
42
|
+
SLOW_RUNNING = "SELECT 1 FROM stage_jobs WHERE pipeline_run_id = pipeline_runs.id " \
|
|
43
|
+
"AND stage = 'slow' AND status = 'running'"
|
|
44
|
+
STALE = "project_path = ? AND branch = ? AND #{ARRIVED_BEFORE} AND (waited_at IS NULL OR waited_at < ?) " \
|
|
45
|
+
"AND NOT EXISTS (#{SLOW_RUNNING}) AND #{ACTIVE}".freeze
|
|
46
|
+
RUN_COLUMNS = "id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid"
|
|
47
|
+
|
|
30
48
|
# The run with this id, when it has not finished; none otherwise.
|
|
31
49
|
def self.with_id(db, id) = where(db, "id = ?", id)
|
|
32
50
|
|
|
33
|
-
# [job id, run id, pid] for each slow stage still running
|
|
34
|
-
# forked process that runs it.
|
|
51
|
+
# [job id, run id, pid] for each slow stage still running or waiting its
|
|
52
|
+
# turn (AT-9.19), pid being the forked process that runs it.
|
|
35
53
|
def self.slow_suites(db)
|
|
36
54
|
db.execute("SELECT stage_jobs.id, pipeline_runs.id, pipeline_runs.pid FROM stage_jobs " \
|
|
37
55
|
"JOIN pipeline_runs ON pipeline_runs.id = stage_jobs.pipeline_run_id " \
|
|
38
|
-
"WHERE stage_jobs.stage = 'slow' AND stage_jobs
|
|
56
|
+
"WHERE stage_jobs.stage = 'slow' AND stage_jobs.#{ACTIVE} " \
|
|
39
57
|
"AND pipeline_runs.pid IS NOT NULL")
|
|
40
58
|
end
|
|
41
59
|
|
|
42
|
-
# Records the slow stage failed,
|
|
60
|
+
# Records the slow stage failed, numbered after every other stage of its
|
|
61
|
+
# run that finished, and that its process died, unless its result was
|
|
62
|
+
# recorded meanwhile.
|
|
43
63
|
def self.slow_suite_died(db, job_id)
|
|
44
|
-
db.execute("UPDATE stage_jobs SET status = 'failed', completed_at =
|
|
45
|
-
|
|
64
|
+
db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?, " \
|
|
65
|
+
"finished_order = #{StageJob::NEXT_IN_RUN}, evidence = ? WHERE id = ? AND #{ACTIVE}",
|
|
66
|
+
[Time.now.utc.iso8601(3), Died.evidence("stage"), job_id])
|
|
46
67
|
end
|
|
47
68
|
|
|
48
69
|
def self.cancelled(db, run)
|
|
@@ -51,20 +72,31 @@ module FunCi
|
|
|
51
72
|
PipelineRun.update_status(db, run.id, "cancelled")
|
|
52
73
|
end
|
|
53
74
|
|
|
75
|
+
# The process groups of the run's stages still going, and its trunk
|
|
76
|
+
# fetch's, whatever the run's own status says.
|
|
77
|
+
def self.groups(db, id)
|
|
78
|
+
fetch = db.execute("SELECT fetch_pgid FROM pipeline_runs WHERE id = ?", [id]).flatten.compact
|
|
79
|
+
[*stage_groups(db, id), *fetch]
|
|
80
|
+
end
|
|
81
|
+
|
|
54
82
|
def self.where(db, clause, *values)
|
|
55
|
-
db.execute("SELECT
|
|
83
|
+
db.execute("SELECT #{RUN_COLUMNS} FROM pipeline_runs " \
|
|
56
84
|
"WHERE #{clause} AND #{ACTIVE} ORDER BY id", values).map { |row| active_run(db, row) }
|
|
57
85
|
end
|
|
58
86
|
private_class_method :where
|
|
59
87
|
|
|
60
88
|
# The groups are each running stage's, and the trunk fetch's (architecture.md, Checking against the trunk).
|
|
61
89
|
def self.active_run(db, (id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid))
|
|
62
|
-
groups = db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
|
|
63
|
-
[id]).flatten
|
|
64
90
|
ActiveRun.new(id: id, commit_hash: commit_hash, processes: [trigger_pid, pid].compact,
|
|
65
|
-
stage_groups: [*
|
|
91
|
+
stage_groups: [*stage_groups(db, id), *fetch_pgid], slot_lock: slot_lock)
|
|
66
92
|
end
|
|
67
93
|
private_class_method :active_run
|
|
94
|
+
|
|
95
|
+
def self.stage_groups(db, id)
|
|
96
|
+
db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
|
|
97
|
+
[id]).flatten
|
|
98
|
+
end
|
|
99
|
+
private_class_method :stage_groups
|
|
68
100
|
end
|
|
69
101
|
end
|
|
70
102
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Persistence
|
|
5
|
+
# When a commit arrived on its branch: its first run there. A commit run
|
|
6
|
+
# again (by a waiter, or `fun-ci trigger` by hand) keeps its place, so it
|
|
7
|
+
# supersedes no commit that arrived after it, and takes no branch's row
|
|
8
|
+
# from one (acceptance-tests.md, AT-9.20).
|
|
9
|
+
module Arrival
|
|
10
|
+
# The first run of the commit of the row `pipeline_runs` names, in SQL.
|
|
11
|
+
FIRST_RUN = "(SELECT MIN(arrived.id) FROM pipeline_runs arrived " \
|
|
12
|
+
"WHERE arrived.project_path IS pipeline_runs.project_path " \
|
|
13
|
+
"AND arrived.branch = pipeline_runs.branch AND arrived.commit_hash = pipeline_runs.commit_hash)"
|
|
14
|
+
# Later than any run: the place of a commit with no run yet.
|
|
15
|
+
NOT_YET = 9_223_372_036_854_775_807
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Persistence
|
|
7
|
+
# Which branches the console keeps out of view, and how (design.md, The
|
|
8
|
+
# console): `ignored`, until it is shown again, or `snoozed`, until a run
|
|
9
|
+
# of the branch starts after the snooze. Like JobVisibility, it says
|
|
10
|
+
# nothing of whether a branch runs, and the agent commands don't read it.
|
|
11
|
+
class BranchVisibility
|
|
12
|
+
# A run is kept to the second and a snooze to the millisecond, so they
|
|
13
|
+
# are compared as times; a run in the second of the snooze doesn't end it.
|
|
14
|
+
HIDDEN = <<~SQL
|
|
15
|
+
SELECT project_path, branch, kind FROM hidden_branches AS hidden WHERE kind = 'ignored' OR NOT EXISTS (
|
|
16
|
+
SELECT 1 FROM pipeline_runs AS run WHERE run.project_path = hidden.project_path
|
|
17
|
+
AND run.branch = hidden.branch
|
|
18
|
+
AND julianday(run.created_at) > julianday(hidden.hidden_at))
|
|
19
|
+
SQL
|
|
20
|
+
|
|
21
|
+
def initialize(db)
|
|
22
|
+
@db = db
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Keeps the project's `branch` out of view from `at`, as `kind`:
|
|
26
|
+
# `snoozed` or `ignored`. The latest of two decides.
|
|
27
|
+
def hide(project, branch, kind, at:)
|
|
28
|
+
@db.execute("INSERT OR REPLACE INTO hidden_branches (project_path, branch, kind, hidden_at) " \
|
|
29
|
+
"VALUES (?, ?, ?, ?)", [project, branch, kind, at.utc.iso8601(3)])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def show(project, branch)
|
|
33
|
+
@db.execute("DELETE FROM hidden_branches WHERE project_path = ? AND branch = ?", [project, branch])
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# { [project, branch] => kind } of the branches out of view now.
|
|
37
|
+
def hidden = @db.execute(HIDDEN).to_h { |project, branch, kind| [[project, branch], kind] }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "sqlite3"
|
|
4
|
+
require_relative "stage_job"
|
|
5
|
+
require_relative "hidden_row_tables"
|
|
4
6
|
|
|
5
7
|
module FunCi
|
|
6
8
|
module Persistence
|
|
@@ -73,21 +75,45 @@ module FunCi
|
|
|
73
75
|
%w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
|
|
74
76
|
%w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
|
|
75
77
|
%w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
|
|
76
|
-
%w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER]
|
|
78
|
+
%w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER],
|
|
79
|
+
%w[job_runs claimed_at TEXT], %w[job_runs began_at TEXT]].freeze
|
|
77
80
|
|
|
78
|
-
TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE
|
|
81
|
+
TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE,
|
|
82
|
+
*HiddenRowTables::ALL].freeze
|
|
83
|
+
|
|
84
|
+
# Where a commit arrived on its branch, which the board and the canceller
|
|
85
|
+
# look up for every run they read (Persistence::Arrival); runs are never
|
|
86
|
+
# deleted, so without it each lookup scans them all.
|
|
87
|
+
ARRIVAL_INDEX = "CREATE INDEX IF NOT EXISTS pipeline_runs_arrival " \
|
|
88
|
+
"ON pipeline_runs (project_path, branch, commit_hash)"
|
|
79
89
|
|
|
80
90
|
def self.migrate!(db)
|
|
81
91
|
with_setup_lock(db.filename("main")) do
|
|
82
92
|
TABLES.each { |table| db.execute(table) }
|
|
83
93
|
ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
|
|
94
|
+
db.execute(ARRIVAL_INDEX)
|
|
84
95
|
db.execute(NAME_DETACHED)
|
|
96
|
+
number_failures_kept_unnumbered(db)
|
|
85
97
|
end
|
|
86
98
|
end
|
|
87
99
|
|
|
88
100
|
# Runs started on a detached HEAD before 2.0.1 were kept with no branch name.
|
|
89
101
|
NAME_DETACHED = "UPDATE pipeline_runs SET branch = 'detached' WHERE branch = ''"
|
|
90
102
|
|
|
103
|
+
# In 2.4.0 and earlier, a slow stage whose process died was kept failed
|
|
104
|
+
# with no finish order, which a verdict needs to tell which failure came
|
|
105
|
+
# first.
|
|
106
|
+
FAILURES_KEPT_UNNUMBERED = "SELECT id FROM stage_jobs WHERE finished_order IS NULL " \
|
|
107
|
+
"AND status IN ('failed', 'timed_out') ORDER BY completed_at, id"
|
|
108
|
+
|
|
109
|
+
# Numbers each after the rest of its run, in the order they ended.
|
|
110
|
+
def self.number_failures_kept_unnumbered(db)
|
|
111
|
+
db.execute(FAILURES_KEPT_UNNUMBERED).each do |(id)|
|
|
112
|
+
db.execute("UPDATE stage_jobs SET finished_order = #{StageJob::NEXT_IN_RUN} WHERE id = ?", [id])
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
private_class_method :number_failures_kept_unnumbered
|
|
116
|
+
|
|
91
117
|
def self.with_setup_lock(db_path)
|
|
92
118
|
File.open("#{db_path}.setup-lock", File::RDWR | File::CREAT, 0o644) do |lock|
|
|
93
119
|
lock.flock(File::LOCK_EX)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../evidence/document"
|
|
5
|
+
|
|
6
|
+
module FunCi
|
|
7
|
+
module Persistence
|
|
8
|
+
# The evidence kept of a stage or job run whose process died: that it has
|
|
9
|
+
# no result, and why that may be, so it reads apart from a failure of the
|
|
10
|
+
# code. It keeps no end, since when it was found dead says nothing of how
|
|
11
|
+
# long it ran.
|
|
12
|
+
module Died
|
|
13
|
+
def self.evidence(what)
|
|
14
|
+
fact = { name: "ended", extractor: "fun-ci",
|
|
15
|
+
value: "its process stopped before it said how the #{what} ended (the machine slept or " \
|
|
16
|
+
"restarted, or something killed it), so this #{what} has no result" }
|
|
17
|
+
JSON.generate(Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [fact]).to_h)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module FunCi
|
|
4
|
+
module Persistence
|
|
5
|
+
# The tables where the console keeps the rows the user hid, per machine
|
|
6
|
+
# (architecture.md, Daily and weekly jobs): JobVisibility's and BranchVisibility's.
|
|
7
|
+
module HiddenRowTables
|
|
8
|
+
JOBS = <<~SQL
|
|
9
|
+
CREATE TABLE IF NOT EXISTS hidden_jobs (
|
|
10
|
+
project_path TEXT, job TEXT, kind TEXT, hidden_at TEXT,
|
|
11
|
+
PRIMARY KEY (project_path, job)
|
|
12
|
+
)
|
|
13
|
+
SQL
|
|
14
|
+
|
|
15
|
+
BRANCHES = <<~SQL
|
|
16
|
+
CREATE TABLE IF NOT EXISTS hidden_branches (
|
|
17
|
+
project_path TEXT, branch TEXT, kind TEXT, hidden_at TEXT, PRIMARY KEY (project_path, branch)
|
|
18
|
+
)
|
|
19
|
+
SQL
|
|
20
|
+
|
|
21
|
+
ALL = [JOBS, BRANCHES].freeze
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -11,7 +11,7 @@ module FunCi
|
|
|
11
11
|
# and weekly jobs), apart from its pipeline runs.
|
|
12
12
|
class JobRuns
|
|
13
13
|
COLUMNS = %i[id project_path job cadence commit_hash branch status started_at completed_at lock_file pid
|
|
14
|
-
group_pid exit_status signal output_tail evidence budget].freeze
|
|
14
|
+
group_pid exit_status signal output_tail evidence budget claimed_at].freeze
|
|
15
15
|
SELECT = "SELECT #{COLUMNS.join(", ")} FROM job_runs".freeze
|
|
16
16
|
KEPT = 10
|
|
17
17
|
|
|
@@ -24,12 +24,14 @@ module FunCi
|
|
|
24
24
|
# and answers its id, when the job is due at `now`; nil otherwise. The
|
|
25
25
|
# job's latest run is read and the new one written in one transaction,
|
|
26
26
|
# so of two claims at once only one starts it. A claim forgets the
|
|
27
|
-
# job's runs beyond its newest 10.
|
|
28
|
-
|
|
27
|
+
# job's runs beyond its newest 10. A block is given the id inside that
|
|
28
|
+
# transaction, so what it records of the run (that it waits its turn)
|
|
29
|
+
# is there the moment anyone else can read the run.
|
|
30
|
+
def claim(job, commit:, lock_file:, now:, &then_record)
|
|
29
31
|
@db.transaction(:immediate) do
|
|
30
32
|
next nil unless Jobs::Due.new(latest(job.name), job.period, now: now).now?
|
|
31
33
|
|
|
32
|
-
insert(job, commit, lock_file, now).tap {
|
|
34
|
+
insert(job, commit, lock_file, now).tap { |id| claimed(job.name, id, then_record) }
|
|
33
35
|
end
|
|
34
36
|
end
|
|
35
37
|
|
|
@@ -53,13 +55,21 @@ module FunCi
|
|
|
53
55
|
|
|
54
56
|
def insert(job, commit, lock_file, now)
|
|
55
57
|
@db.execute("INSERT INTO job_runs (project_path, job, cadence, commit_hash, branch, status, started_at, " \
|
|
56
|
-
"lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)",
|
|
57
|
-
[@project, job.name, job.cadence, commit[:sha], commit[:branch], stamp(now), lock_file])
|
|
58
|
+
"claimed_at, began_at, lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, ?)",
|
|
59
|
+
[@project, job.name, job.cadence, commit[:sha], commit[:branch], *[stamp(now)] * 3, lock_file])
|
|
58
60
|
@db.last_insert_row_id
|
|
59
61
|
end
|
|
60
62
|
|
|
63
|
+
def claimed(name, id, then_record)
|
|
64
|
+
forget_beyond(name)
|
|
65
|
+
then_record&.call(id)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# A run began once it runs: began_at says when, and is none while it waits its turn.
|
|
61
69
|
def update(id, status, at, was)
|
|
62
|
-
|
|
70
|
+
began = stamp(at) if status == "running"
|
|
71
|
+
@db.execute("UPDATE job_runs SET status = ?, started_at = ?, began_at = ? WHERE id = ? AND #{was}",
|
|
72
|
+
[status, stamp(at), began, id])
|
|
63
73
|
@db.changes
|
|
64
74
|
end
|
|
65
75
|
|