fun_ci 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +73 -0
  3. data/README.md +16 -7
  4. data/lib/fun_ci/agent/events.rb +15 -7
  5. data/lib/fun_ci/agent/job_events.rb +12 -5
  6. data/lib/fun_ci/agent/onward.rb +21 -0
  7. data/lib/fun_ci/agent/output.rb +7 -0
  8. data/lib/fun_ci/agent/reports.rb +3 -0
  9. data/lib/fun_ci/agent/snapshots.rb +25 -7
  10. data/lib/fun_ci/agent/status_text.rb +7 -3
  11. data/lib/fun_ci/agent/wait_command.rb +37 -5
  12. data/lib/fun_ci/console/board_data.rb +46 -11
  13. data/lib/fun_ci/console/branch_page.rb +9 -0
  14. data/lib/fun_ci/console/console_state.rb +22 -8
  15. data/lib/fun_ci/console/cursor.rb +22 -0
  16. data/lib/fun_ci/console/cursor_rows.rb +37 -0
  17. data/lib/fun_ci/console/job_message.rb +5 -4
  18. data/lib/fun_ci/console/job_order.rb +5 -3
  19. data/lib/fun_ci/console/job_row.rb +17 -0
  20. data/lib/fun_ci/console/job_rows.rb +14 -3
  21. data/lib/fun_ci/console/key_handler.rb +88 -51
  22. data/lib/fun_ci/console/question.rb +63 -0
  23. data/lib/fun_ci/console/run_message.rb +3 -3
  24. data/lib/fun_ci/console/view.rb +20 -6
  25. data/lib/fun_ci/jobs/job_run.rb +10 -9
  26. data/lib/fun_ci/persistence/active_runs.rb +47 -17
  27. data/lib/fun_ci/persistence/arrival.rb +18 -0
  28. data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
  29. data/lib/fun_ci/persistence/database.rb +28 -2
  30. data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
  31. data/lib/fun_ci/persistence/job_runs.rb +17 -7
  32. data/lib/fun_ci/persistence/job_visibility.rb +43 -0
  33. data/lib/fun_ci/persistence/null_recorder.rb +3 -0
  34. data/lib/fun_ci/persistence/pipeline_recorder.rb +22 -3
  35. data/lib/fun_ci/persistence/pipeline_run.rb +15 -4
  36. data/lib/fun_ci/persistence/stage_job.rb +13 -0
  37. data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
  38. data/lib/fun_ci/pipeline/process_runner.rb +8 -7
  39. data/lib/fun_ci/pipeline/run_canceller.rb +1 -3
  40. data/lib/fun_ci/pipeline/slot.rb +12 -0
  41. data/lib/fun_ci/pipeline/slot_run.rb +20 -5
  42. data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
  43. data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
  44. data/lib/fun_ci/pipeline/trigger.rb +9 -1
  45. data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
  46. data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
  47. data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
  48. data/lib/fun_ci/pipeline/worktree_pool.rb +5 -2
  49. data/lib/fun_ci/pipeline/worktrees.rb +3 -0
  50. data/lib/fun_ci/trunk/check.rb +7 -2
  51. data/lib/fun_ci/version.rb +1 -1
  52. metadata +26 -11
@@ -4,6 +4,7 @@ require "time"
4
4
  require_relative "pipeline_run"
5
5
  require_relative "stage_job"
6
6
  require_relative "died"
7
+ require_relative "arrival"
7
8
 
8
9
  module FunCi
9
10
  module Persistence
@@ -20,31 +21,49 @@ module FunCi
20
21
  module ActiveRuns
21
22
  ACTIVE = "status IN ('scheduled', 'running')"
22
23
 
23
- # The runs on the branch a new commit's run may cancel: not the new
24
- # commit's own, nor one an agent has waited on since `waited_before`
25
- # (an ISO 8601 UTC time; acceptance-tests.md, AT-9.12).
26
- def self.cancellable(db, branch, new_commit:, waited_before:)
27
- where(db, "project_path = ? AND branch = ? AND commit_hash != ? AND (waited_at IS NULL OR waited_at < ?)",
28
- branch.project, branch.name, new_commit, waited_before)
24
+ # Records cancelled, and returns, the runs on the branch a new commit's
25
+ # run cancels: those of commits that arrived before it (acceptance-tests.md,
26
+ # AT-9.20), except one an agent has waited on since `waited_before` (an
27
+ # ISO 8601 UTC time; AT-9.12) and one whose slow suite is running (AT-9.19).
28
+ # Picked and recorded in one statement, before their processes are
29
+ # stopped, so a slow suite whose turn comes meanwhile never starts
30
+ # (StageJob.started_in_turn?).
31
+ def self.cancel_stale(db, branch, new_commit:, waited_before:)
32
+ values = [Time.now.utc.iso8601, branch.project, branch.name, branch.project, branch.name, new_commit,
33
+ waited_before]
34
+ rows = db.execute("UPDATE pipeline_runs SET status = 'cancelled', updated_at = ? WHERE #{STALE} " \
35
+ "RETURNING #{RUN_COLUMNS}", values)
36
+ rows.sort_by(&:first).map { |row| active_run(db, row) }
29
37
  end
30
38
 
39
+ # A commit arrives on its branch with its first run; a commit with none yet arrives last.
40
+ ARRIVED_BEFORE = "#{Arrival::FIRST_RUN} < COALESCE((SELECT MIN(id) FROM pipeline_runs " \
41
+ "WHERE project_path IS ? AND branch = ? AND commit_hash = ?), #{Arrival::NOT_YET})".freeze
42
+ SLOW_RUNNING = "SELECT 1 FROM stage_jobs WHERE pipeline_run_id = pipeline_runs.id " \
43
+ "AND stage = 'slow' AND status = 'running'"
44
+ STALE = "project_path = ? AND branch = ? AND #{ARRIVED_BEFORE} AND (waited_at IS NULL OR waited_at < ?) " \
45
+ "AND NOT EXISTS (#{SLOW_RUNNING}) AND #{ACTIVE}".freeze
46
+ RUN_COLUMNS = "id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid"
47
+
31
48
  # The run with this id, when it has not finished; none otherwise.
32
49
  def self.with_id(db, id) = where(db, "id = ?", id)
33
50
 
34
- # [job id, run id, pid] for each slow stage still running, pid being the
35
- # forked process that runs it.
51
+ # [job id, run id, pid] for each slow stage still running or waiting its
52
+ # turn (AT-9.19), pid being the forked process that runs it.
36
53
  def self.slow_suites(db)
37
54
  db.execute("SELECT stage_jobs.id, pipeline_runs.id, pipeline_runs.pid FROM stage_jobs " \
38
55
  "JOIN pipeline_runs ON pipeline_runs.id = stage_jobs.pipeline_run_id " \
39
- "WHERE stage_jobs.stage = 'slow' AND stage_jobs.status = 'running' " \
56
+ "WHERE stage_jobs.stage = 'slow' AND stage_jobs.#{ACTIVE} " \
40
57
  "AND pipeline_runs.pid IS NOT NULL")
41
58
  end
42
59
 
43
- # Records the slow stage failed, and that its process died, unless its
44
- # result was recorded meanwhile.
60
+ # Records the slow stage failed, numbered after every other stage of its
61
+ # run that finished, and that its process died, unless its result was
62
+ # recorded meanwhile.
45
63
  def self.slow_suite_died(db, job_id)
46
- db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?, evidence = ? " \
47
- "WHERE id = ? AND status = 'running'", [Time.now.utc.iso8601(3), Died.evidence("stage"), job_id])
64
+ db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?, " \
65
+ "finished_order = #{StageJob::NEXT_IN_RUN}, evidence = ? WHERE id = ? AND #{ACTIVE}",
66
+ [Time.now.utc.iso8601(3), Died.evidence("stage"), job_id])
48
67
  end
49
68
 
50
69
  def self.cancelled(db, run)
@@ -53,20 +72,31 @@ module FunCi
53
72
  PipelineRun.update_status(db, run.id, "cancelled")
54
73
  end
55
74
 
75
+ # The process groups of the run's stages still going, and its trunk
76
+ # fetch's, whatever the run's own status says.
77
+ def self.groups(db, id)
78
+ fetch = db.execute("SELECT fetch_pgid FROM pipeline_runs WHERE id = ?", [id]).flatten.compact
79
+ [*stage_groups(db, id), *fetch]
80
+ end
81
+
56
82
  def self.where(db, clause, *values)
57
- db.execute("SELECT id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid FROM pipeline_runs " \
83
+ db.execute("SELECT #{RUN_COLUMNS} FROM pipeline_runs " \
58
84
  "WHERE #{clause} AND #{ACTIVE} ORDER BY id", values).map { |row| active_run(db, row) }
59
85
  end
60
86
  private_class_method :where
61
87
 
62
88
  # The groups are each running stage's, and the trunk fetch's (architecture.md, Checking against the trunk).
63
89
  def self.active_run(db, (id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid))
64
- groups = db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
65
- [id]).flatten
66
90
  ActiveRun.new(id: id, commit_hash: commit_hash, processes: [trigger_pid, pid].compact,
67
- stage_groups: [*groups, *fetch_pgid], slot_lock: slot_lock)
91
+ stage_groups: [*stage_groups(db, id), *fetch_pgid], slot_lock: slot_lock)
68
92
  end
69
93
  private_class_method :active_run
94
+
95
+ def self.stage_groups(db, id)
96
+ db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
97
+ [id]).flatten
98
+ end
99
+ private_class_method :stage_groups
70
100
  end
71
101
  end
72
102
  end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Persistence
5
+ # When a commit arrived on its branch: its first run there. A commit run
6
+ # again (by a waiter, or `fun-ci trigger` by hand) keeps its place, so it
7
+ # supersedes no commit that arrived after it, and takes no branch's row
8
+ # from one (acceptance-tests.md, AT-9.20).
9
+ module Arrival
10
+ # The first run of the commit of the row `pipeline_runs` names, in SQL.
11
+ FIRST_RUN = "(SELECT MIN(arrived.id) FROM pipeline_runs arrived " \
12
+ "WHERE arrived.project_path IS pipeline_runs.project_path " \
13
+ "AND arrived.branch = pipeline_runs.branch AND arrived.commit_hash = pipeline_runs.commit_hash)"
14
+ # Later than any run: the place of a commit with no run yet.
15
+ NOT_YET = 9_223_372_036_854_775_807
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # Which branches the console keeps out of view, and how (design.md, The
8
+ # console): `ignored`, until it is shown again, or `snoozed`, until a run
9
+ # of the branch starts after the snooze. Like JobVisibility, it says
10
+ # nothing of whether a branch runs, and the agent commands don't read it.
11
+ class BranchVisibility
12
+ # A run is kept to the second and a snooze to the millisecond, so they
13
+ # are compared as times; a run in the second of the snooze doesn't end it.
14
+ HIDDEN = <<~SQL
15
+ SELECT project_path, branch, kind FROM hidden_branches AS hidden WHERE kind = 'ignored' OR NOT EXISTS (
16
+ SELECT 1 FROM pipeline_runs AS run WHERE run.project_path = hidden.project_path
17
+ AND run.branch = hidden.branch
18
+ AND julianday(run.created_at) > julianday(hidden.hidden_at))
19
+ SQL
20
+
21
+ def initialize(db)
22
+ @db = db
23
+ end
24
+
25
+ # Keeps the project's `branch` out of view from `at`, as `kind`:
26
+ # `snoozed` or `ignored`. The latest of two decides.
27
+ def hide(project, branch, kind, at:)
28
+ @db.execute("INSERT OR REPLACE INTO hidden_branches (project_path, branch, kind, hidden_at) " \
29
+ "VALUES (?, ?, ?, ?)", [project, branch, kind, at.utc.iso8601(3)])
30
+ end
31
+
32
+ def show(project, branch)
33
+ @db.execute("DELETE FROM hidden_branches WHERE project_path = ? AND branch = ?", [project, branch])
34
+ end
35
+
36
+ # { [project, branch] => kind } of the branches out of view now.
37
+ def hidden = @db.execute(HIDDEN).to_h { |project, branch, kind| [[project, branch], kind] }
38
+ end
39
+ end
40
+ end
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sqlite3"
4
+ require_relative "stage_job"
5
+ require_relative "hidden_row_tables"
4
6
 
5
7
  module FunCi
6
8
  module Persistence
@@ -73,21 +75,45 @@ module FunCi
73
75
  %w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
74
76
  %w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
75
77
  %w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
76
- %w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER]].freeze
78
+ %w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER],
79
+ %w[job_runs claimed_at TEXT], %w[job_runs began_at TEXT]].freeze
77
80
 
78
- TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE].freeze
81
+ TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE,
82
+ *HiddenRowTables::ALL].freeze
83
+
84
+ # Where a commit arrived on its branch, which the board and the canceller
85
+ # look up for every run they read (Persistence::Arrival); runs are never
86
+ # deleted, so without it each lookup scans them all.
87
+ ARRIVAL_INDEX = "CREATE INDEX IF NOT EXISTS pipeline_runs_arrival " \
88
+ "ON pipeline_runs (project_path, branch, commit_hash)"
79
89
 
80
90
  def self.migrate!(db)
81
91
  with_setup_lock(db.filename("main")) do
82
92
  TABLES.each { |table| db.execute(table) }
83
93
  ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
94
+ db.execute(ARRIVAL_INDEX)
84
95
  db.execute(NAME_DETACHED)
96
+ number_failures_kept_unnumbered(db)
85
97
  end
86
98
  end
87
99
 
88
100
  # Runs started on a detached HEAD before 2.0.1 were kept with no branch name.
89
101
  NAME_DETACHED = "UPDATE pipeline_runs SET branch = 'detached' WHERE branch = ''"
90
102
 
103
+ # In 2.4.0 and earlier, a slow stage whose process died was kept failed
104
+ # with no finish order, which a verdict needs to tell which failure came
105
+ # first.
106
+ FAILURES_KEPT_UNNUMBERED = "SELECT id FROM stage_jobs WHERE finished_order IS NULL " \
107
+ "AND status IN ('failed', 'timed_out') ORDER BY completed_at, id"
108
+
109
+ # Numbers each after the rest of its run, in the order they ended.
110
+ def self.number_failures_kept_unnumbered(db)
111
+ db.execute(FAILURES_KEPT_UNNUMBERED).each do |(id)|
112
+ db.execute("UPDATE stage_jobs SET finished_order = #{StageJob::NEXT_IN_RUN} WHERE id = ?", [id])
113
+ end
114
+ end
115
+ private_class_method :number_failures_kept_unnumbered
116
+
91
117
  def self.with_setup_lock(db_path)
92
118
  File.open("#{db_path}.setup-lock", File::RDWR | File::CREAT, 0o644) do |lock|
93
119
  lock.flock(File::LOCK_EX)
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Persistence
5
+ # The tables where the console keeps the rows the user hid, per machine
6
+ # (architecture.md, Daily and weekly jobs): JobVisibility's and BranchVisibility's.
7
+ module HiddenRowTables
8
+ JOBS = <<~SQL
9
+ CREATE TABLE IF NOT EXISTS hidden_jobs (
10
+ project_path TEXT, job TEXT, kind TEXT, hidden_at TEXT,
11
+ PRIMARY KEY (project_path, job)
12
+ )
13
+ SQL
14
+
15
+ BRANCHES = <<~SQL
16
+ CREATE TABLE IF NOT EXISTS hidden_branches (
17
+ project_path TEXT, branch TEXT, kind TEXT, hidden_at TEXT, PRIMARY KEY (project_path, branch)
18
+ )
19
+ SQL
20
+
21
+ ALL = [JOBS, BRANCHES].freeze
22
+ end
23
+ end
24
+ end
@@ -11,7 +11,7 @@ module FunCi
11
11
  # and weekly jobs), apart from its pipeline runs.
12
12
  class JobRuns
13
13
  COLUMNS = %i[id project_path job cadence commit_hash branch status started_at completed_at lock_file pid
14
- group_pid exit_status signal output_tail evidence budget].freeze
14
+ group_pid exit_status signal output_tail evidence budget claimed_at].freeze
15
15
  SELECT = "SELECT #{COLUMNS.join(", ")} FROM job_runs".freeze
16
16
  KEPT = 10
17
17
 
@@ -24,12 +24,14 @@ module FunCi
24
24
  # and answers its id, when the job is due at `now`; nil otherwise. The
25
25
  # job's latest run is read and the new one written in one transaction,
26
26
  # so of two claims at once only one starts it. A claim forgets the
27
- # job's runs beyond its newest 10.
28
- def claim(job, commit:, lock_file:, now:)
27
+ # job's runs beyond its newest 10. A block is given the id inside that
28
+ # transaction, so what it records of the run (that it waits its turn)
29
+ # is there the moment anyone else can read the run.
30
+ def claim(job, commit:, lock_file:, now:, &then_record)
29
31
  @db.transaction(:immediate) do
30
32
  next nil unless Jobs::Due.new(latest(job.name), job.period, now: now).now?
31
33
 
32
- insert(job, commit, lock_file, now).tap { forget_beyond(job.name) }
34
+ insert(job, commit, lock_file, now).tap { |id| claimed(job.name, id, then_record) }
33
35
  end
34
36
  end
35
37
 
@@ -53,13 +55,21 @@ module FunCi
53
55
 
54
56
  def insert(job, commit, lock_file, now)
55
57
  @db.execute("INSERT INTO job_runs (project_path, job, cadence, commit_hash, branch, status, started_at, " \
56
- "lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)",
57
- [@project, job.name, job.cadence, commit[:sha], commit[:branch], stamp(now), lock_file])
58
+ "claimed_at, began_at, lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?, ?, ?)",
59
+ [@project, job.name, job.cadence, commit[:sha], commit[:branch], *[stamp(now)] * 3, lock_file])
58
60
  @db.last_insert_row_id
59
61
  end
60
62
 
63
+ def claimed(name, id, then_record)
64
+ forget_beyond(name)
65
+ then_record&.call(id)
66
+ end
67
+
68
+ # A run began once it runs: began_at says when, and is none while it waits its turn.
61
69
  def update(id, status, at, was)
62
- @db.execute("UPDATE job_runs SET status = ?, started_at = ? WHERE id = ? AND #{was}", [status, stamp(at), id])
70
+ began = stamp(at) if status == "running"
71
+ @db.execute("UPDATE job_runs SET status = ?, started_at = ?, began_at = ? WHERE id = ? AND #{was}",
72
+ [status, stamp(at), began, id])
63
73
  @db.changes
64
74
  end
65
75
 
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # Which daily and weekly jobs the console keeps out of view, and how
8
+ # (design.md, The console): `ignored`, until it is shown again, or
9
+ # `snoozed`, until the job runs again. It says nothing of whether a job
10
+ # runs, and the agent commands don't read it.
11
+ class JobVisibility
12
+ # When a run began, if it did: not while it waits its turn, nor when
13
+ # cancelled before it (design.md, Daily and weekly jobs), and once it
14
+ # did, whatever became of it. fun-ci 2.4.0 and earlier kept no
15
+ # `began_at`, so of their runs those that ran or are running began when they started.
16
+ BEGAN = "COALESCE(run.began_at, CASE WHEN run.status IN ('running', 'completed', 'failed', 'timed_out') " \
17
+ "THEN run.started_at END)"
18
+ HIDDEN = <<~SQL.freeze
19
+ SELECT job, kind FROM hidden_jobs AS hidden WHERE project_path = ? AND (kind = 'ignored' OR NOT EXISTS (
20
+ SELECT 1 FROM job_runs AS run WHERE run.project_path = hidden.project_path AND run.job = hidden.job
21
+ AND #{BEGAN} > hidden.hidden_at))
22
+ SQL
23
+
24
+ def initialize(db)
25
+ @db = db
26
+ end
27
+
28
+ # Keeps the project's job `name` out of view from `at`, as `kind`:
29
+ # `snoozed` or `ignored`. The latest of two decides.
30
+ def hide(project, name, kind, at:)
31
+ @db.execute("INSERT OR REPLACE INTO hidden_jobs (project_path, job, kind, hidden_at) VALUES (?, ?, ?, ?)",
32
+ [project, name, kind, at.utc.iso8601(3)])
33
+ end
34
+
35
+ def show(project, name)
36
+ @db.execute("DELETE FROM hidden_jobs WHERE project_path = ? AND job = ?", [project, name])
37
+ end
38
+
39
+ # { name => kind } of the project's jobs out of view now.
40
+ def hidden(project) = @db.execute(HIDDEN, [project]).to_h
41
+ end
42
+ end
43
+ end
@@ -8,6 +8,9 @@ module FunCi
8
8
  class NullRecorder
9
9
  def create_run(**); end
10
10
  def start_stage(_stage, **); end
11
+ def schedule_stage(_stage, **); end
12
+ def started_in_turn?(_job_id) = true
13
+ def arrival_place = 0
11
14
  def end_stage(_job_id, _status); end
12
15
  def keep_exit(_job_id, _exit_status, _signal); end
13
16
  def keep_evidence(_job_id, _document); end
@@ -71,16 +71,31 @@ module FunCi
71
71
  end
72
72
 
73
73
  def start_stage(stage, budget: nil)
74
+ schedule_stage(stage, budget: budget).tap { |job_id| stage_started(job_id) if job_id }
75
+ end
76
+
77
+ # A stage that waits before it starts, as a slow suite does for its turn.
78
+ def schedule_stage(stage, budget: nil)
74
79
  return nil unless @pipeline_run_id
75
80
 
76
81
  tolerating do
77
82
  ensure_running
78
- job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
79
- StageJob.update_status(@db, job_id, "running")
80
- job_id
83
+ StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
81
84
  end
82
85
  end
83
86
 
87
+ # Records a slow suite started once its turn comes; false when its run
88
+ # was cancelled while it waited. One whose stage could not be recorded,
89
+ # or whose start the database refused, starts.
90
+ def started_in_turn?(job_id)
91
+ return true unless job_id
92
+
93
+ tolerating { StageJob.started_in_turn?(@db, job_id) } != false
94
+ end
95
+
96
+ # Where this run's commit arrived on its branch (Persistence::Arrival); 0 when not known.
97
+ def arrival_place = (@pipeline_run_id && tolerating { PipelineRun.arrival_place(@db, @pipeline_run_id) }) || 0
98
+
84
99
  # Keeps a failed stage's evidence, and the output's tail where an older
85
100
  # fun-ci sharing the database reads it.
86
101
  def keep_evidence(job_id, document)
@@ -113,6 +128,10 @@ module FunCi
113
128
 
114
129
  private
115
130
 
131
+ def stage_started(job_id)
132
+ tolerating { StageJob.update_status(@db, job_id, "running") }
133
+ end
134
+
116
135
  def ensure_running
117
136
  run = PipelineRun.find(@db, @pipeline_run_id)
118
137
  return unless run && run[:status] == "scheduled"
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "time"
4
+ require_relative "arrival"
4
5
 
5
6
  module FunCi
6
7
  module Persistence
@@ -56,6 +57,11 @@ module FunCi
56
57
  db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
57
58
  end
58
59
 
60
+ # The id of the first run of this run's commit on its branch: where the commit arrived.
61
+ def self.arrival_place(db, id)
62
+ db.get_first_value("SELECT #{Arrival::FIRST_RUN} FROM pipeline_runs WHERE id = ?", [id])
63
+ end
64
+
59
65
  def self.update_status(db, id, new_status, at: Time.now)
60
66
  db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, at.utc.iso8601, id])
61
67
  end
@@ -69,7 +75,10 @@ module FunCi
69
75
  "WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
70
76
 
71
77
  # Each branch's newest run.
72
- HEADS = "SELECT MAX(id) FROM pipeline_runs GROUP BY project_path, branch"
78
+ # Each branch's head: the newest run of the commit that arrived on it last (Arrival).
79
+ HEADS = "SELECT MAX(id) FROM pipeline_runs WHERE #{Arrival::FIRST_RUN} IN " \
80
+ "(SELECT MAX(#{Arrival::FIRST_RUN}) FROM pipeline_runs GROUP BY project_path, branch) " \
81
+ "GROUP BY project_path, branch".freeze
73
82
 
74
83
  # The newest run of each of `limit` branches, of the `among` whose newest
75
84
  # run changed most recently: those that most need you, then those run
@@ -79,9 +88,11 @@ module FunCi
79
88
  "ORDER BY updated_at DESC, id DESC LIMIT ?) ORDER BY #{NEEDING}, id DESC LIMIT ?", among, limit)
80
89
  end
81
90
 
82
- # A project's branch's runs, newest first.
83
- def self.of_branch(db, project_path, branch, limit:)
84
- query(db, "WHERE project_path IS ? AND branch = ? ORDER BY id DESC LIMIT ?", project_path, branch, limit)
91
+ # The runs of `head`'s branch up to it, newest first: not a run of an
92
+ # older commit run again since.
93
+ def self.leading_to(db, head, limit:)
94
+ query(db, "WHERE project_path IS ? AND branch = ? AND id <= ? ORDER BY id DESC LIMIT ?",
95
+ head[:project_path], head[:branch], head[:id], limit)
85
96
  end
86
97
 
87
98
  def self.query(db, clause, *params)
@@ -22,6 +22,19 @@ module FunCi
22
22
  db.last_insert_row_id
23
23
  end
24
24
 
25
+ # Records a slow suite waiting its branch's turn running, unless it or its
26
+ # run was recorded cancelled meanwhile; whether it was. One statement, so
27
+ # a cancel recorded before it refuses it and one after finds it running
28
+ # (acceptance-tests.md, AT-9.19).
29
+ def self.started_in_turn?(db, id)
30
+ db.execute("UPDATE stage_jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'scheduled' " \
31
+ "AND NOT EXISTS (#{RUN_CANCELLED})", [Time.now.utc.iso8601(3), id])
32
+ db.changes == 1
33
+ end
34
+
35
+ RUN_CANCELLED = "SELECT 1 FROM pipeline_runs WHERE pipeline_runs.id = stage_jobs.pipeline_run_id " \
36
+ "AND pipeline_runs.status = 'cancelled'"
37
+
25
38
  # The stage script's process, which leads a process group of its own.
26
39
  def self.store_pid(db, id, pid)
27
40
  db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
@@ -3,6 +3,7 @@
3
3
  require "json"
4
4
  require "time"
5
5
  require_relative "../trunk/check"
6
+ require_relative "pipeline_run"
6
7
 
7
8
  module FunCi
8
9
  module Persistence
@@ -12,7 +13,7 @@ module FunCi
12
13
  TIP = %i[trunk_remote trunk_branch trunk_sha trunk_seen_at].freeze
13
14
  MERGE = %i[outcome ahead behind files reason].freeze
14
15
  COLUMNS = [:commit_hash, *TIP, *MERGE].freeze
15
- SELECT = "SELECT #{COLUMNS.join(", ")} FROM trunk_checks".freeze
16
+ SELECT = "SELECT #{COLUMNS.join(", ")}, checked_at FROM trunk_checks".freeze
16
17
 
17
18
  def initialize(db, project)
18
19
  @db = db
@@ -36,14 +37,14 @@ module FunCi
36
37
  found = @db.execute("#{SELECT} WHERE project_path = ? AND commit_hash = ? " \
37
38
  "ORDER BY COALESCE(trunk_seen_at, checked_at) DESC, id DESC LIMIT 1", [@project, sha]).first
38
39
  found && Trunk::Check.new(commit: found.first, tip: tip(found[1, TIP.size]),
39
- merge: merge(found.last(MERGE.size)))
40
+ merge: merge(found[1 + TIP.size, MERGE.size]), checked_at: Time.parse(found.last))
40
41
  end
41
42
 
42
- # { commit => trunk SHA } for each branch's newest run, other than `except`,
43
- # whose commit has been checked against a trunk tip.
43
+ # { commit => trunk SHA } for each branch's head (PipelineRun::HEADS), other
44
+ # than `except`, whose commit has been checked against a trunk tip.
44
45
  def branch_heads(except:)
45
- heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (SELECT MAX(id) FROM pipeline_runs " \
46
- "WHERE project_path = ? GROUP BY branch) AND commit_hash != ?", [@project, except]).flatten
46
+ heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (#{PipelineRun::HEADS}) " \
47
+ "AND project_path = ? AND commit_hash != ?", [@project, except]).flatten
47
48
  heads.to_h { |sha| [sha, latest(sha)&.tip&.sha] }.compact
48
49
  end
49
50
 
@@ -29,13 +29,14 @@ module FunCi
29
29
  # fun-ci's own, the window what it prints is written to, what to do
30
30
  # with its pid when it runs over budget, before it is killed, the
31
31
  # open files it holds as long as it runs, such as a lock, and what it
32
- # is run under (a Priorities prefix, "" for fun-ci's own priority).
33
- Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority)
32
+ # is run under (a Priorities prefix, "" for fun-ci's own priority), and
33
+ # how long its output may drain once it is killed.
34
+ Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority, :drain)
34
35
 
35
36
  class Launch
36
37
  def initialize(**given)
37
38
  super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
38
- **given)
39
+ drain: DRAIN_SECONDS, **given)
39
40
  end
40
41
  end
41
42
 
@@ -100,12 +101,12 @@ module FunCi
100
101
  def over_budget(pid, reading, launch)
101
102
  launch.before_kill&.call(pid)
102
103
  kill_process_group(pid)
103
- [drained(reading, launch.output), nil, true]
104
+ [drained(reading, launch), nil, true]
104
105
  end
105
106
 
106
- def drained(reading, printed)
107
- reading.join(DRAIN_SECONDS)
108
- text(printed)
107
+ def drained(reading, launch)
108
+ reading.join(launch.drain)
109
+ text(launch.output)
109
110
  end
110
111
 
111
112
  def text(window) = window.text.force_encoding(Encoding.default_external)
@@ -81,9 +81,7 @@ module FunCi
81
81
  # records a stage's group before it lets the stage start, so once those
82
82
  # processes are dead, looking again finds every stage they started.
83
83
  def stop_late_stages(db, run)
84
- Persistence::ActiveRuns.with_id(db, run.id).each do |now|
85
- (now.stage_groups - run.stage_groups).each { |group| kill(-group) }
86
- end
84
+ (Persistence::ActiveRuns.groups(db, run.id) - run.stage_groups).each { |group| kill(-group) }
87
85
  end
88
86
 
89
87
  # The groups recorded since `before` was read.
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "waiting_slots"
4
+
3
5
  module FunCi
4
6
  module Pipeline
5
7
  # A worktree a pipeline has to itself, held through an flock on its lock
@@ -26,6 +28,16 @@ module FunCi
26
28
  @lock.close if @holders.zero? && !@lock.closed?
27
29
  end
28
30
 
31
+ # Marks the slot as held only by a slow suite waiting its turn, until
32
+ # the mark is closed (WaitingSlots). A slot with nothing to lock has
33
+ # nothing to mark.
34
+ def waiting = lock_file ? WaitingSlots.mark(path) : Unmarked.new
35
+
36
+ # The mark of a slot with nothing to lock: closing it does nothing.
37
+ class Unmarked
38
+ def close; end
39
+ end
40
+
29
41
  # Whether its worktree was just made, so has no caches yet.
30
42
  def cold? = false
31
43
 
@@ -5,6 +5,7 @@ require_relative "background_fork"
5
5
  require_relative "stage_runner"
6
6
  require_relative "stage_execution"
7
7
  require_relative "progress_reporter"
8
+ require_relative "turn_queue"
8
9
 
9
10
  module FunCi
10
11
  module Pipeline
@@ -71,26 +72,40 @@ module FunCi
71
72
  end
72
73
 
73
74
  def launch_slow_suite(config)
74
- job_id = recorder.start_stage("slow", budget: @seams.budgets["slow"])
75
+ job_id = recorder.schedule_stage("slow", budget: @seams.budgets["slow"])
75
76
  launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
76
77
  executor: slow_suite(config, @slot.share))
77
78
  end
78
79
 
79
80
  # Called with the recorder and job to record the slow suite with, in
80
- # whichever process runs it; the slot is let go once it is recorded.
81
+ # whichever process runs it, once no older slow suite of the branch runs
82
+ # (acceptance-tests.md, AT-9.19); the slot is let go once it is recorded.
81
83
  def slow_suite(config, slot)
82
84
  execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
83
85
  launching: { priority: @seams.priorities.slow })
84
86
  command = config.stage_command("slow", @commit.sha)
85
- ->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
87
+ ->(recorder, job_id) { in_turn(slot, recorder, job_id) { execution.run("slow", command, recorder, job_id) } }
86
88
  end
87
89
 
88
- def holding(slot)
89
- yield
90
+ # Holding the slot until the slow suite is recorded, whether or not its
91
+ # turn came, marked waiting until it does (WaitingSlots); a run
92
+ # cancelled while it waited never starts it.
93
+ def in_turn(slot, recorder, job_id, &)
94
+ waiting = slot.waiting
95
+ take_turn(waiting, recorder, job_id, &)
90
96
  ensure
97
+ waiting&.close
91
98
  slot.release
92
99
  end
93
100
 
101
+ def take_turn(waiting, recorder, job_id)
102
+ ticket = TurnTicket.new(place: recorder.arrival_place, id: job_id.to_i)
103
+ @seams.slow_turns.take(@commit.branch, ticket) do
104
+ waiting.close
105
+ yield if recorder.started_in_turn?(job_id)
106
+ end
107
+ end
108
+
94
109
  def launch(**)
95
110
  return @seams.background_launcher.call(**) if @seams.background_launcher
96
111