fun_ci 2.3.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +143 -0
  3. data/README.md +20 -10
  4. data/lib/fun_ci/agent/events.rb +15 -7
  5. data/lib/fun_ci/agent/job_events.rb +12 -5
  6. data/lib/fun_ci/agent/onward.rb +21 -0
  7. data/lib/fun_ci/agent/output.rb +7 -0
  8. data/lib/fun_ci/agent/reports.rb +3 -0
  9. data/lib/fun_ci/agent/snapshots.rb +25 -7
  10. data/lib/fun_ci/agent/status_text.rb +9 -5
  11. data/lib/fun_ci/agent/wait_command.rb +37 -5
  12. data/lib/fun_ci/cli.rb +2 -0
  13. data/lib/fun_ci/cli_command_help.rb +51 -0
  14. data/lib/fun_ci/cli_help.rb +9 -9
  15. data/lib/fun_ci/console/board_data.rb +50 -10
  16. data/lib/fun_ci/console/branch_page.rb +9 -0
  17. data/lib/fun_ci/console/console_state.rb +22 -8
  18. data/lib/fun_ci/console/cursor.rb +22 -0
  19. data/lib/fun_ci/console/cursor_rows.rb +37 -0
  20. data/lib/fun_ci/console/job_message.rb +5 -4
  21. data/lib/fun_ci/console/job_order.rb +5 -3
  22. data/lib/fun_ci/console/job_row.rb +17 -0
  23. data/lib/fun_ci/console/job_rows.rb +14 -3
  24. data/lib/fun_ci/console/key_handler.rb +88 -51
  25. data/lib/fun_ci/console/question.rb +63 -0
  26. data/lib/fun_ci/console/run_message.rb +3 -3
  27. data/lib/fun_ci/console/trunk_marks.rb +3 -1
  28. data/lib/fun_ci/console/view.rb +20 -6
  29. data/lib/fun_ci/evidence/about_stage.rb +1 -1
  30. data/lib/fun_ci/evidence/collector.rb +6 -3
  31. data/lib/fun_ci/evidence/outcome.rb +6 -3
  32. data/lib/fun_ci/jobs/job_run.rb +10 -9
  33. data/lib/fun_ci/jobs/locks.rb +1 -1
  34. data/lib/fun_ci/persistence/active_jobs.rb +2 -11
  35. data/lib/fun_ci/persistence/active_runs.rb +48 -16
  36. data/lib/fun_ci/persistence/arrival.rb +18 -0
  37. data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
  38. data/lib/fun_ci/persistence/database.rb +28 -2
  39. data/lib/fun_ci/persistence/died.rb +21 -0
  40. data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
  41. data/lib/fun_ci/persistence/job_recorder.rb +1 -0
  42. data/lib/fun_ci/persistence/job_runs.rb +17 -7
  43. data/lib/fun_ci/persistence/job_visibility.rb +43 -0
  44. data/lib/fun_ci/persistence/null_recorder.rb +4 -0
  45. data/lib/fun_ci/persistence/pipeline_recorder.rb +23 -3
  46. data/lib/fun_ci/persistence/pipeline_run.rb +25 -11
  47. data/lib/fun_ci/persistence/stage_job.rb +25 -0
  48. data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
  49. data/lib/fun_ci/pipeline/group_kill.rb +38 -0
  50. data/lib/fun_ci/pipeline/pipeline_forker.rb +3 -1
  51. data/lib/fun_ci/pipeline/process_runner.rb +14 -8
  52. data/lib/fun_ci/pipeline/process_tree.rb +37 -0
  53. data/lib/fun_ci/pipeline/progress_reporter.rb +2 -1
  54. data/lib/fun_ci/pipeline/run_canceller.rb +16 -6
  55. data/lib/fun_ci/pipeline/slot.rb +12 -0
  56. data/lib/fun_ci/pipeline/slot_run.rb +20 -5
  57. data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
  58. data/lib/fun_ci/pipeline/stage_end.rb +2 -1
  59. data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
  60. data/lib/fun_ci/pipeline/trigger.rb +9 -1
  61. data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
  62. data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
  63. data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
  64. data/lib/fun_ci/pipeline/workspaces.rb +2 -1
  65. data/lib/fun_ci/pipeline/worktree_pool.rb +27 -7
  66. data/lib/fun_ci/pipeline/worktree_prune.rb +9 -4
  67. data/lib/fun_ci/pipeline/worktrees.rb +41 -7
  68. data/lib/fun_ci/setup/commands.rb +16 -7
  69. data/lib/fun_ci/setup/eslint_detection.rb +38 -0
  70. data/lib/fun_ci/setup/init_options.rb +34 -0
  71. data/lib/fun_ci/setup/installer.rb +41 -7
  72. data/lib/fun_ci/setup/placeholders.rb +30 -0
  73. data/lib/fun_ci/setup/project_config.rb +1 -0
  74. data/lib/fun_ci/setup/settings.rb +21 -1
  75. data/lib/fun_ci/setup/setup_checker.rb +5 -3
  76. data/lib/fun_ci/setup/stage_templates/javascript.rb +14 -4
  77. data/lib/fun_ci/setup/stage_templates/placeholders.rb +21 -0
  78. data/lib/fun_ci/setup/stage_templates.rb +6 -4
  79. data/lib/fun_ci/setup/template_writer.rb +3 -5
  80. data/lib/fun_ci/trunk/check.rb +7 -2
  81. data/lib/fun_ci/version.rb +1 -1
  82. metadata +34 -11
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module FunCi
6
+ module Persistence
7
+ # Which daily and weekly jobs the console keeps out of view, and how
8
+ # (design.md, The console): `ignored`, until it is shown again, or
9
+ # `snoozed`, until the job runs again. It says nothing of whether a job
10
+ # runs, and the agent commands don't read it.
11
+ class JobVisibility
12
+ # When a run began, if it did: not while it waits its turn, nor when
13
+ # cancelled before it (design.md, Daily and weekly jobs), and once it
14
+ # did, whatever became of it. fun-ci 2.4.0 and earlier kept no
15
+ # `began_at`, so of their runs those that ran or are running began when they started.
16
+ BEGAN = "COALESCE(run.began_at, CASE WHEN run.status IN ('running', 'completed', 'failed', 'timed_out') " \
17
+ "THEN run.started_at END)"
18
+ HIDDEN = <<~SQL.freeze
19
+ SELECT job, kind FROM hidden_jobs AS hidden WHERE project_path = ? AND (kind = 'ignored' OR NOT EXISTS (
20
+ SELECT 1 FROM job_runs AS run WHERE run.project_path = hidden.project_path AND run.job = hidden.job
21
+ AND #{BEGAN} > hidden.hidden_at))
22
+ SQL
23
+
24
+ def initialize(db)
25
+ @db = db
26
+ end
27
+
28
+ # Keeps the project's job `name` out of view from `at`, as `kind`:
29
+ # `snoozed` or `ignored`. The latest of two decides.
30
+ def hide(project, name, kind, at:)
31
+ @db.execute("INSERT OR REPLACE INTO hidden_jobs (project_path, job, kind, hidden_at) VALUES (?, ?, ?, ?)",
32
+ [project, name, kind, at.utc.iso8601(3)])
33
+ end
34
+
35
+ def show(project, name)
36
+ @db.execute("DELETE FROM hidden_jobs WHERE project_path = ? AND job = ?", [project, name])
37
+ end
38
+
39
+ # { name => kind } of the project's jobs out of view now.
40
+ def hidden(project) = @db.execute(HIDDEN, [project]).to_h
41
+ end
42
+ end
43
+ end
@@ -8,6 +8,9 @@ module FunCi
8
8
  class NullRecorder
9
9
  def create_run(**); end
10
10
  def start_stage(_stage, **); end
11
+ def schedule_stage(_stage, **); end
12
+ def started_in_turn?(_job_id) = true
13
+ def arrival_place = 0
11
14
  def end_stage(_job_id, _status); end
12
15
  def keep_exit(_job_id, _exit_status, _signal); end
13
16
  def keep_evidence(_job_id, _document); end
@@ -19,6 +22,7 @@ module FunCi
19
22
  def trunk_heads(**) = {}
20
23
  def trunk_fetches = TrunkRecording::NoFetches.new
21
24
  def alongside(_job_id) = []
25
+ def alongside_elsewhere(_job_id) = []
22
26
  def stage_process(_job_id, _pid); end
23
27
  def slot_taken(_lock_file); end
24
28
  def foreground_done; end
@@ -71,16 +71,31 @@ module FunCi
71
71
  end
72
72
 
73
73
  def start_stage(stage, budget: nil)
74
+ schedule_stage(stage, budget: budget).tap { |job_id| stage_started(job_id) if job_id }
75
+ end
76
+
77
+ # A stage that waits before it starts, as a slow suite does for its turn.
78
+ def schedule_stage(stage, budget: nil)
74
79
  return nil unless @pipeline_run_id
75
80
 
76
81
  tolerating do
77
82
  ensure_running
78
- job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
79
- StageJob.update_status(@db, job_id, "running")
80
- job_id
83
+ StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
81
84
  end
82
85
  end
83
86
 
87
+ # Records a slow suite started once its turn comes; false when its run
88
+ # was cancelled while it waited. One whose stage could not be recorded,
89
+ # or whose start the database refused, starts.
90
+ def started_in_turn?(job_id)
91
+ return true unless job_id
92
+
93
+ tolerating { StageJob.started_in_turn?(@db, job_id) } != false
94
+ end
95
+
96
+ # Where this run's commit arrived on its branch (Persistence::Arrival); 0 when not known.
97
+ def arrival_place = (@pipeline_run_id && tolerating { PipelineRun.arrival_place(@db, @pipeline_run_id) }) || 0
98
+
84
99
  # Keeps a failed stage's evidence, and the output's tail where an older
85
100
  # fun-ci sharing the database reads it.
86
101
  def keep_evidence(job_id, document)
@@ -97,6 +112,7 @@ module FunCi
97
112
 
98
113
  # The stages that shared the slot with this one (architecture.md, "Evidence of a failed stage").
99
114
  def alongside(job_id) = tolerating { StageJob.alongside(@db, job_id) } || []
115
+ def alongside_elsewhere(job_id) = tolerating { StageJob.alongside_elsewhere(@db, job_id) } || []
100
116
 
101
117
  def keep_exit(job_id, exit_status, signal)
102
118
  tolerating { StageJob.keep_exit(@db, job_id, exit_status, signal) }
@@ -112,6 +128,10 @@ module FunCi
112
128
 
113
129
  private
114
130
 
131
+ def stage_started(job_id)
132
+ tolerating { StageJob.update_status(@db, job_id, "running") }
133
+ end
134
+
115
135
  def ensure_running
116
136
  run = PipelineRun.find(@db, @pipeline_run_id)
117
137
  return unless run && run[:status] == "scheduled"
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "time"
4
+ require_relative "arrival"
4
5
 
5
6
  module FunCi
6
7
  module Persistence
@@ -56,9 +57,13 @@ module FunCi
56
57
  db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
57
58
  end
58
59
 
59
- def self.update_status(db, id, new_status)
60
- now = Time.now.utc.iso8601
61
- db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, now, id])
60
+ # The id of the first run of this run's commit on its branch: where the commit arrived.
61
+ def self.arrival_place(db, id)
62
+ db.get_first_value("SELECT #{Arrival::FIRST_RUN} FROM pipeline_runs WHERE id = ?", [id])
63
+ end
64
+
65
+ def self.update_status(db, id, new_status, at: Time.now)
66
+ db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, at.utc.iso8601, id])
62
67
  end
63
68
 
64
69
  def self.recent(db, limit: 20)
@@ -69,16 +74,25 @@ module FunCi
69
74
  NEEDING = "CASE status WHEN 'failed' THEN 0 WHEN 'timed_out' THEN 1 WHEN 'running' THEN 2 " \
70
75
  "WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
71
76
 
72
- # The newest run of each of `limit` branches: those that most need you,
73
- # then those run most recently, in that order.
74
- def self.branch_heads(db, limit:)
75
- query(db, "WHERE id IN (SELECT MAX(id) FROM pipeline_runs GROUP BY project_path, branch) " \
76
- "ORDER BY #{NEEDING}, id DESC LIMIT ?", limit)
77
+ # Each branch's newest run.
78
+ # Each branch's head: the newest run of the commit that arrived on it last (Arrival).
79
+ HEADS = "SELECT MAX(id) FROM pipeline_runs WHERE #{Arrival::FIRST_RUN} IN " \
80
+ "(SELECT MAX(#{Arrival::FIRST_RUN}) FROM pipeline_runs GROUP BY project_path, branch) " \
81
+ "GROUP BY project_path, branch".freeze
82
+
83
+ # The newest run of each of `limit` branches, of the `among` whose newest
84
+ # run changed most recently: those that most need you, then those run
85
+ # most recently, in that order.
86
+ def self.branch_heads(db, limit:, among:)
87
+ query(db, "WHERE id IN (SELECT id FROM pipeline_runs WHERE id IN (#{HEADS}) " \
88
+ "ORDER BY updated_at DESC, id DESC LIMIT ?) ORDER BY #{NEEDING}, id DESC LIMIT ?", among, limit)
77
89
  end
78
90
 
79
- # A project's branch's runs, newest first.
80
- def self.of_branch(db, project_path, branch, limit:)
81
- query(db, "WHERE project_path IS ? AND branch = ? ORDER BY id DESC LIMIT ?", project_path, branch, limit)
91
+ # The runs of `head`'s branch up to it, newest first: not a run of an
92
+ # older commit run again since.
93
+ def self.leading_to(db, head, limit:)
94
+ query(db, "WHERE project_path IS ? AND branch = ? AND id <= ? ORDER BY id DESC LIMIT ?",
95
+ head[:project_path], head[:branch], head[:id], limit)
82
96
  end
83
97
 
84
98
  def self.query(db, clause, *params)
@@ -22,6 +22,19 @@ module FunCi
22
22
  db.last_insert_row_id
23
23
  end
24
24
 
25
+ # Records a slow suite waiting its branch's turn running, unless it or its
26
+ # run was recorded cancelled meanwhile; whether it was. One statement, so
27
+ # a cancel recorded before it refuses it and one after finds it running
28
+ # (acceptance-tests.md, AT-9.19).
29
+ def self.started_in_turn?(db, id)
30
+ db.execute("UPDATE stage_jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'scheduled' " \
31
+ "AND NOT EXISTS (#{RUN_CANCELLED})", [Time.now.utc.iso8601(3), id])
32
+ db.changes == 1
33
+ end
34
+
35
+ RUN_CANCELLED = "SELECT 1 FROM pipeline_runs WHERE pipeline_runs.id = stage_jobs.pipeline_run_id " \
36
+ "AND pipeline_runs.status = 'cancelled'"
37
+
25
38
  # The stage script's process, which leads a process group of its own.
26
39
  def self.store_pid(db, id, pid)
27
40
  db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
@@ -51,6 +64,18 @@ module FunCi
51
64
  [id]).flatten
52
65
  end
53
66
 
67
+ # Stages of the project's other runs running at any moment between this
68
+ # stage's start and now, as "<short commit>'s <stage>".
69
+ def self.alongside_elsewhere(db, id)
70
+ db.execute("SELECT runs.commit_hash, others.stage FROM stage_jobs AS others " \
71
+ "JOIN pipeline_runs AS runs ON runs.id = others.pipeline_run_id " \
72
+ "JOIN stage_jobs AS me ON me.id = ? JOIN pipeline_runs AS mine ON mine.id = me.pipeline_run_id " \
73
+ "WHERE runs.project_path IS mine.project_path AND runs.id != mine.id " \
74
+ "AND others.started_at IS NOT NULL " \
75
+ "AND (others.completed_at IS NULL OR others.completed_at > me.started_at) ORDER BY others.id",
76
+ [id]).map { |sha, stage| "#{sha[0, 7]}'s #{stage}" }
77
+ end
78
+
54
79
  def self.ids(db) = db.execute("SELECT id FROM stage_jobs").flatten
55
80
 
56
81
  def self.find(db, id)
@@ -3,6 +3,7 @@
3
3
  require "json"
4
4
  require "time"
5
5
  require_relative "../trunk/check"
6
+ require_relative "pipeline_run"
6
7
 
7
8
  module FunCi
8
9
  module Persistence
@@ -12,7 +13,7 @@ module FunCi
12
13
  TIP = %i[trunk_remote trunk_branch trunk_sha trunk_seen_at].freeze
13
14
  MERGE = %i[outcome ahead behind files reason].freeze
14
15
  COLUMNS = [:commit_hash, *TIP, *MERGE].freeze
15
- SELECT = "SELECT #{COLUMNS.join(", ")} FROM trunk_checks".freeze
16
+ SELECT = "SELECT #{COLUMNS.join(", ")}, checked_at FROM trunk_checks".freeze
16
17
 
17
18
  def initialize(db, project)
18
19
  @db = db
@@ -36,14 +37,14 @@ module FunCi
36
37
  found = @db.execute("#{SELECT} WHERE project_path = ? AND commit_hash = ? " \
37
38
  "ORDER BY COALESCE(trunk_seen_at, checked_at) DESC, id DESC LIMIT 1", [@project, sha]).first
38
39
  found && Trunk::Check.new(commit: found.first, tip: tip(found[1, TIP.size]),
39
- merge: merge(found.last(MERGE.size)))
40
+ merge: merge(found[1 + TIP.size, MERGE.size]), checked_at: Time.parse(found.last))
40
41
  end
41
42
 
42
- # { commit => trunk SHA } for each branch's newest run, other than `except`,
43
- # whose commit has been checked against a trunk tip.
43
+ # { commit => trunk SHA } for each branch's head (PipelineRun::HEADS), other
44
+ # than `except`, whose commit has been checked against a trunk tip.
44
45
  def branch_heads(except:)
45
- heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (SELECT MAX(id) FROM pipeline_runs " \
46
- "WHERE project_path = ? GROUP BY branch) AND commit_hash != ?", [@project, except]).flatten
46
+ heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (#{PipelineRun::HEADS}) " \
47
+ "AND project_path = ? AND commit_hash != ?", [@project, except]).flatten
47
48
  heads.to_h { |sha| [sha, latest(sha)&.tip&.sha] }.compact
48
49
  end
49
50
 
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Pipeline
5
+ # Kills a process group and what left it (ProcessTree): TERM, then KILL.
6
+ # A member forking as the KILL arrives leaves a child in the group that
7
+ # the signal missed, so the group is killed again while any member lives,
8
+ # a bounded number of times, since a mutant may stop it ever going.
9
+ module GroupKill
10
+ ROUNDS = 10
11
+
12
+ # escaped: further targets, -pgid for a group, pid for a process.
13
+ def self.kill(pid, escaped, signal: Process.method(:kill))
14
+ %w[TERM KILL].product([-pid, *escaped]).each { |name, target| send_to(signal, name, target) }
15
+ ROUNDS.times do
16
+ return unless lives?(pid, signal)
17
+
18
+ send_to(signal, "KILL", -pid)
19
+ end
20
+ end
21
+
22
+ # On macOS a group left with only zombies answers EPERM, not ESRCH.
23
+ def self.lives?(pid, signal)
24
+ signal.call(0, -pid)
25
+ true
26
+ rescue Errno::ESRCH, Errno::EPERM
27
+ false
28
+ end
29
+
30
+ def self.send_to(signal, name, target)
31
+ signal.call(name, target)
32
+ rescue Errno::ESRCH, Errno::EPERM
33
+ # Gone already.
34
+ end
35
+ private_class_method :lives?, :send_to
36
+ end
37
+ end
38
+ end
@@ -66,8 +66,10 @@ module FunCi
66
66
  end
67
67
 
68
68
  # Trigger requires this file, through TriggerCommand, so it is loaded
69
- # here, where it is used, rather than above.
69
+ # here, where it is used, rather than above. The run leads a session of
70
+ # its own, so that ending the shell that committed doesn't end it.
70
71
  def self.run_in_child(commit_hash:, branch:, db_path:)
72
+ Process.setsid
71
73
  require_relative "trigger"
72
74
  recorder = Persistence::DbRecorder.new(Persistence::Database.connection(db_path))
73
75
  trigger = trigger(commit_hash, branch, recorder)
@@ -3,6 +3,9 @@
3
3
  require_relative "gate"
4
4
  require_relative "git_environment"
5
5
  require_relative "output_window"
6
+ require_relative "process_tree"
7
+ require_relative "group_kill"
8
+ require_relative "../evidence/process_table"
6
9
 
7
10
  module FunCi
8
11
  module Pipeline
@@ -26,13 +29,14 @@ module FunCi
26
29
  # fun-ci's own, the window what it prints is written to, what to do
27
30
  # with its pid when it runs over budget, before it is killed, the
28
31
  # open files it holds as long as it runs, such as a lock, and what it
29
- # is run under (a Priorities prefix, "" for fun-ci's own priority).
30
- Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority)
32
+ # is run under (a Priorities prefix, "" for fun-ci's own priority), and
33
+ # how long its output may drain once it is killed.
34
+ Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority, :drain)
31
35
 
32
36
  class Launch
33
37
  def initialize(**given)
34
38
  super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
35
- **given)
39
+ drain: DRAIN_SECONDS, **given)
36
40
  end
37
41
  end
38
42
 
@@ -97,12 +101,12 @@ module FunCi
97
101
  def over_budget(pid, reading, launch)
98
102
  launch.before_kill&.call(pid)
99
103
  kill_process_group(pid)
100
- [drained(reading, launch.output), nil, true]
104
+ [drained(reading, launch), nil, true]
101
105
  end
102
106
 
103
- def drained(reading, printed)
104
- reading.join(DRAIN_SECONDS)
105
- text(printed)
107
+ def drained(reading, launch)
108
+ reading.join(launch.drain)
109
+ text(launch.output)
106
110
  end
107
111
 
108
112
  def text(window) = window.text.force_encoding(Encoding.default_external)
@@ -112,8 +116,10 @@ module FunCi
112
116
  [output, status, false]
113
117
  end
114
118
 
119
+ # With the group go what it started that left it (ProcessTree), found
120
+ # before the kill orphans them (GroupKill).
115
121
  def kill_process_group(pid)
116
- %w[TERM KILL].each { |signal| ignoring_errors { Process.kill(signal, -pid) } }
122
+ GroupKill.kill(pid, ProcessTree.escaped([pid], Evidence::ProcessTable.now))
117
123
  ignoring_errors { Process.waitpid(pid) }
118
124
  end
119
125
 
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FunCi
4
+ module Pipeline
5
+ # What to kill besides a stage's process groups: each process descended
6
+ # from one of their members, by parent id, that has left for a group or
7
+ # session of its own, as Node's detached children do. The rows must be
8
+ # read before the kill, since a killed parent's children move to init.
9
+ # A double-forked daemon has moved to init already and is not reached.
10
+ module ProcessTree
11
+ # Signal targets: -pid for an escapee leading its own group, which
12
+ # takes that group with it, and the pid alone of one in any other group.
13
+ def self.escaped(groups, rows)
14
+ escapees = outside(groups, descendants(in_groups(groups, rows), rows))
15
+ leaders = escapees.select { |row| row.pgid == row.pid }.map(&:pid)
16
+ leaders.map(&:-@) + escapees.reject { |row| leaders.include?(row.pgid) }.map(&:pid)
17
+ end
18
+
19
+ def self.in_groups(groups, rows) = rows.select { |row| groups.include?(row.pgid) }
20
+ def self.outside(groups, rows) = rows.reject { |row| groups.include?(row.pgid) }
21
+
22
+ # Bounded by the table's size, so a table whose parent ids form a cycle still ends.
23
+ def self.descendants(found, rows)
24
+ rows.size.times do
25
+ more = children(found.map(&:pid), rows) - found
26
+ return found if more.empty?
27
+
28
+ found += more
29
+ end
30
+ found
31
+ end
32
+
33
+ def self.children(parents, rows) = rows.select { |row| parents.include?(row.ppid) }
34
+ private_class_method :in_groups, :outside, :descendants, :children
35
+ end
36
+ end
37
+ end
@@ -18,7 +18,8 @@ module FunCi
18
18
  end
19
19
 
20
20
  def cold_slot(seconds)
21
- @stdout.puts "fun-ci: a new worktree, whose caches are empty, so lint and build have #{seconds}s this once."
21
+ @stdout.puts "fun-ci: a cold worktree (new, just cleaned, or unused for a day), so lint and build have " \
22
+ "#{seconds}s this once."
22
23
  end
23
24
 
24
25
  def slow_launched
@@ -5,18 +5,24 @@ require_relative "../persistence/active_jobs"
5
5
  require_relative "../persistence/run_status"
6
6
  require_relative "../persistence/write_trouble"
7
7
  require_relative "slot_lock"
8
+ require_relative "process_tree"
9
+ require_relative "../evidence/process_table"
8
10
 
9
11
  module FunCi
10
12
  module Pipeline
11
13
  # Stops every process of a run. The run's own processes go first, so none
12
14
  # of them records the run as failed once its stages die; then each stage's
13
- # process group. A run whose slot nobody holds any more has died already,
15
+ # process group, and last what a stage started that left its group
16
+ # (ProcessTree). A run whose slot nobody holds any more has died already,
14
17
  # and the pids it recorded may belong to other processes now, so it is
15
18
  # left alone; so is this process.
16
19
  class RunCanceller
17
- def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?))
20
+ # process_table: what ps lists now, as Evidence::ProcessTable rows.
21
+ def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?),
22
+ process_table: Evidence::ProcessTable.method(:now))
18
23
  @killer = killer
19
24
  @slot_held = slot_held
25
+ @process_table = process_table
20
26
  end
21
27
 
22
28
  # Stops the run and records it, and its unfinished stages, cancelled.
@@ -60,18 +66,22 @@ module FunCi
60
66
  def stop(run)
61
67
  return unless alive?(run)
62
68
 
63
- (run.processes - [Process.pid] + run.stage_groups.map(&:-@)).each { |pid| kill(pid) }
69
+ (run.processes - [Process.pid] + run.stage_groups.map(&:-@) + escaped(run)).each { |pid| kill(pid) }
64
70
  end
65
71
 
66
72
  private
67
73
 
74
+ def escaped(run)
75
+ return [] if run.stage_groups.empty?
76
+
77
+ ProcessTree.escaped(run.stage_groups, @process_table.call)
78
+ end
79
+
68
80
  # A stage its process started after `run` was read. A run's process
69
81
  # records a stage's group before it lets the stage start, so once those
70
82
  # processes are dead, looking again finds every stage they started.
71
83
  def stop_late_stages(db, run)
72
- Persistence::ActiveRuns.with_id(db, run.id).each do |now|
73
- (now.stage_groups - run.stage_groups).each { |group| kill(-group) }
74
- end
84
+ (Persistence::ActiveRuns.groups(db, run.id) - run.stage_groups).each { |group| kill(-group) }
75
85
  end
76
86
 
77
87
  # The groups recorded since `before` was read.
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "waiting_slots"
4
+
3
5
  module FunCi
4
6
  module Pipeline
5
7
  # A worktree a pipeline has to itself, held through an flock on its lock
@@ -26,6 +28,16 @@ module FunCi
26
28
  @lock.close if @holders.zero? && !@lock.closed?
27
29
  end
28
30
 
31
+ # Marks the slot as held only by a slow suite waiting its turn, until
32
+ # the mark is closed (WaitingSlots). A slot with nothing to lock has
33
+ # nothing to mark.
34
+ def waiting = lock_file ? WaitingSlots.mark(path) : Unmarked.new
35
+
36
+ # The mark of a slot with nothing to lock: closing it does nothing.
37
+ class Unmarked
38
+ def close; end
39
+ end
40
+
29
41
  # Whether its worktree was just made, so has no caches yet.
30
42
  def cold? = false
31
43
 
@@ -5,6 +5,7 @@ require_relative "background_fork"
5
5
  require_relative "stage_runner"
6
6
  require_relative "stage_execution"
7
7
  require_relative "progress_reporter"
8
+ require_relative "turn_queue"
8
9
 
9
10
  module FunCi
10
11
  module Pipeline
@@ -71,26 +72,40 @@ module FunCi
71
72
  end
72
73
 
73
74
  def launch_slow_suite(config)
74
- job_id = recorder.start_stage("slow", budget: @seams.budgets["slow"])
75
+ job_id = recorder.schedule_stage("slow", budget: @seams.budgets["slow"])
75
76
  launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
76
77
  executor: slow_suite(config, @slot.share))
77
78
  end
78
79
 
79
80
  # Called with the recorder and job to record the slow suite with, in
80
- # whichever process runs it; the slot is let go once it is recorded.
81
+ # whichever process runs it, once no older slow suite of the branch runs
82
+ # (acceptance-tests.md, AT-9.19); the slot is let go once it is recorded.
81
83
  def slow_suite(config, slot)
82
84
  execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
83
85
  launching: { priority: @seams.priorities.slow })
84
86
  command = config.stage_command("slow", @commit.sha)
85
- ->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
87
+ ->(recorder, job_id) { in_turn(slot, recorder, job_id) { execution.run("slow", command, recorder, job_id) } }
86
88
  end
87
89
 
88
- def holding(slot)
89
- yield
90
+ # Holding the slot until the slow suite is recorded, whether or not its
91
+ # turn came, marked waiting until it does (WaitingSlots); a run
92
+ # cancelled while it waited never starts it.
93
+ def in_turn(slot, recorder, job_id, &)
94
+ waiting = slot.waiting
95
+ take_turn(waiting, recorder, job_id, &)
90
96
  ensure
97
+ waiting&.close
91
98
  slot.release
92
99
  end
93
100
 
101
+ def take_turn(waiting, recorder, job_id)
102
+ ticket = TurnTicket.new(place: recorder.arrival_place, id: job_id.to_i)
103
+ @seams.slow_turns.take(@commit.branch, ticket) do
104
+ waiting.close
105
+ yield if recorder.started_in_turn?(job_id)
106
+ end
107
+ end
108
+
94
109
  def launch(**)
95
110
  return @seams.background_launcher.call(**) if @seams.background_launcher
96
111
 
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "fileutils"
5
+ require_relative "turn_queue"
6
+
7
+ module FunCi
8
+ module Pipeline
9
+ # The turns a branch's slow suites take: one runs at a time, and of those
10
+ # waiting, the one whose commit arrived last goes next (acceptance-tests.md,
11
+ # AT-9.19, AT-9.20). A turn is an flock on the branch's file under `root`,
12
+ # and each suite holds a ticket in the branch's TurnQueue while it waits and
13
+ # runs, so both end with the process that held them, a slow suite cancelled
14
+ # while it waits included.
15
+ class SlowTurns
16
+ # The most passes taking a turn makes. Each but the last lets a newer
17
+ # suite run to its end, so this covers more than a branch queues; it
18
+ # stops a pass that never ends, as a mutant's, all the same.
19
+ ROUNDS = 100
20
+
21
+ # wait: called once each pass with the paths of the newer suites'
22
+ # tickets to let go first, none on the last; blocks until those are done.
23
+ def initialize(root, wait: TurnQueue::WAIT)
24
+ @root = root
25
+ @wait = wait
26
+ end
27
+
28
+ # Yields once no other process holds the branch's turn and no newer
29
+ # suite of the branch waits for it, holding the turn until the block returns.
30
+ def take(branch, ticket)
31
+ queue = TurnQueue.new(queue_dir(branch))
32
+ queue.holding(ticket) do
33
+ File.open(lock_path(branch), File::RDWR | File::CREAT, 0o644) do |lock|
34
+ in_order(queue, ticket, lock)
35
+ yield
36
+ end
37
+ end
38
+ end
39
+
40
+ def lock_path(branch) = File.join(@root, "#{digest(branch)}.lock")
41
+ def ticket_path(branch, ticket) = File.join(queue_dir(branch), ticket.name)
42
+
43
+ private
44
+
45
+ # Each newer suite goes first; with the turn held it looks again, as a
46
+ # newer one may have come meanwhile. A wait ends only once its ticket's
47
+ # holder let go, so each ticket is waited for once: one that still seems
48
+ # held is no live suite's, and the passes end with the tickets seen.
49
+ def in_order(queue, ticket, lock)
50
+ waited = Set.new
51
+ ROUNDS.times do
52
+ waited.merge(let_go(queue.newer_than(ticket) - waited))
53
+ return if turn_taken?(queue, ticket, lock, waited)
54
+ end
55
+ lock.flock(File::LOCK_EX)
56
+ end
57
+
58
+ # Takes the turn, and whether no suite newer than those `waited` for
59
+ # came meanwhile; lets the turn go again if one did.
60
+ def turn_taken?(queue, ticket, lock, waited)
61
+ lock.flock(File::LOCK_EX)
62
+ return true if (queue.newer_than(ticket) - waited).empty?
63
+
64
+ lock.flock(File::LOCK_UN)
65
+ false
66
+ end
67
+
68
+ def let_go(tickets) = tickets.tap { @wait.call(tickets) }
69
+
70
+ def queue_dir(branch) = File.join(@root, digest(branch))
71
+ def digest(branch) = Digest::SHA256.hexdigest(branch)[0, 16]
72
+ end
73
+ end
74
+ end
@@ -58,7 +58,8 @@ module FunCi
58
58
  status = finished.status
59
59
  Evidence::Outcome.new(state: STATES.fetch(outcome(finished)), exit_status: status&.exitstatus,
60
60
  signal: status&.termsig && Signal.signame(status.termsig),
61
- alongside: @recorder.alongside(@job_id), overrun: finished.overrun)
61
+ alongside: @recorder.alongside(@job_id),
62
+ elsewhere: @recorder.alongside_elsewhere(@job_id), overrun: finished.overrun)
62
63
  end
63
64
 
64
65
  def keep_exit(status)
@@ -21,7 +21,7 @@ module FunCi
21
21
 
22
22
  def cancel(new_commit_hash:)
23
23
  waited_before = (Time.now - WAITED_ON_SECONDS).utc.iso8601
24
- Persistence::ActiveRuns.cancellable(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
24
+ Persistence::ActiveRuns.cancel_stale(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
25
25
  .each { |run| cancel_run(run, new_commit_hash) }
26
26
  end
27
27