fun_ci 2.4.0-x86_64-linux → 2.5.0-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +73 -0
  3. data/README.md +16 -7
  4. data/lib/fun_ci/agent/events.rb +15 -7
  5. data/lib/fun_ci/agent/job_events.rb +12 -5
  6. data/lib/fun_ci/agent/onward.rb +21 -0
  7. data/lib/fun_ci/agent/output.rb +7 -0
  8. data/lib/fun_ci/agent/reports.rb +3 -0
  9. data/lib/fun_ci/agent/snapshots.rb +25 -7
  10. data/lib/fun_ci/agent/status_text.rb +7 -3
  11. data/lib/fun_ci/agent/wait_command.rb +37 -5
  12. data/lib/fun_ci/console/board_data.rb +46 -11
  13. data/lib/fun_ci/console/branch_page.rb +9 -0
  14. data/lib/fun_ci/console/console_state.rb +22 -8
  15. data/lib/fun_ci/console/cursor.rb +22 -0
  16. data/lib/fun_ci/console/cursor_rows.rb +37 -0
  17. data/lib/fun_ci/console/job_message.rb +5 -4
  18. data/lib/fun_ci/console/job_order.rb +5 -3
  19. data/lib/fun_ci/console/job_row.rb +17 -0
  20. data/lib/fun_ci/console/job_rows.rb +14 -3
  21. data/lib/fun_ci/console/key_handler.rb +88 -51
  22. data/lib/fun_ci/console/question.rb +63 -0
  23. data/lib/fun_ci/console/run_message.rb +3 -3
  24. data/lib/fun_ci/console/view.rb +20 -6
  25. data/lib/fun_ci/jobs/job_run.rb +10 -9
  26. data/lib/fun_ci/persistence/active_runs.rb +47 -17
  27. data/lib/fun_ci/persistence/arrival.rb +18 -0
  28. data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
  29. data/lib/fun_ci/persistence/database.rb +28 -2
  30. data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
  31. data/lib/fun_ci/persistence/job_runs.rb +17 -7
  32. data/lib/fun_ci/persistence/job_visibility.rb +43 -0
  33. data/lib/fun_ci/persistence/null_recorder.rb +3 -0
  34. data/lib/fun_ci/persistence/pipeline_recorder.rb +22 -3
  35. data/lib/fun_ci/persistence/pipeline_run.rb +15 -4
  36. data/lib/fun_ci/persistence/stage_job.rb +13 -0
  37. data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
  38. data/lib/fun_ci/pipeline/process_runner.rb +8 -7
  39. data/lib/fun_ci/pipeline/run_canceller.rb +1 -3
  40. data/lib/fun_ci/pipeline/slot.rb +12 -0
  41. data/lib/fun_ci/pipeline/slot_run.rb +20 -5
  42. data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
  43. data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
  44. data/lib/fun_ci/pipeline/trigger.rb +9 -1
  45. data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
  46. data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
  47. data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
  48. data/lib/fun_ci/pipeline/worktree_pool.rb +5 -2
  49. data/lib/fun_ci/pipeline/worktrees.rb +3 -0
  50. data/lib/fun_ci/trunk/check.rb +7 -2
  51. data/lib/fun_ci/version.rb +1 -1
  52. data/libexec/fun-ci-renderer +0 -0
  53. metadata +25 -10
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "fileutils"
5
+ require_relative "turn_queue"
6
+
7
+ module FunCi
8
+ module Pipeline
9
+ # The turns a branch's slow suites take: one runs at a time, and of those
10
+ # waiting, the one whose commit arrived last goes next (acceptance-tests.md,
11
+ # AT-9.19, AT-9.20). A turn is an flock on the branch's file under `root`,
12
+ # and each suite holds a ticket in the branch's TurnQueue while it waits and
13
+ # runs, so both end with the process that held them, a slow suite cancelled
14
+ # while it waits included.
15
+ class SlowTurns
16
+ # The most passes taking a turn makes. Each but the last lets a newer
17
+ # suite run to its end, so this covers more than a branch queues; it
18
+ # stops a pass that never ends, as a mutant's, all the same.
19
+ ROUNDS = 100
20
+
21
+ # wait: called once each pass with the paths of the newer suites'
22
+ # tickets to let go first, none on the last; blocks until those are done.
23
+ def initialize(root, wait: TurnQueue::WAIT)
24
+ @root = root
25
+ @wait = wait
26
+ end
27
+
28
+ # Yields once no other process holds the branch's turn and no newer
29
+ # suite of the branch waits for it, holding the turn until the block returns.
30
+ def take(branch, ticket)
31
+ queue = TurnQueue.new(queue_dir(branch))
32
+ queue.holding(ticket) do
33
+ File.open(lock_path(branch), File::RDWR | File::CREAT, 0o644) do |lock|
34
+ in_order(queue, ticket, lock)
35
+ yield
36
+ end
37
+ end
38
+ end
39
+
40
+ def lock_path(branch) = File.join(@root, "#{digest(branch)}.lock")
41
+ def ticket_path(branch, ticket) = File.join(queue_dir(branch), ticket.name)
42
+
43
+ private
44
+
45
+ # Each newer suite goes first; with the turn held it looks again, as a
46
+ # newer one may have come meanwhile. A wait ends only once its ticket's
47
+ # holder let go, so each ticket is waited for once: one that still seems
48
+ # held is no live suite's, and the passes end with the tickets seen.
49
+ def in_order(queue, ticket, lock)
50
+ waited = Set.new
51
+ ROUNDS.times do
52
+ waited.merge(let_go(queue.newer_than(ticket) - waited))
53
+ return if turn_taken?(queue, ticket, lock, waited)
54
+ end
55
+ lock.flock(File::LOCK_EX)
56
+ end
57
+
58
+ # Takes the turn, and whether no suite newer than those `waited` for
59
+ # came meanwhile; lets the turn go again if one did.
60
+ def turn_taken?(queue, ticket, lock, waited)
61
+ lock.flock(File::LOCK_EX)
62
+ return true if (queue.newer_than(ticket) - waited).empty?
63
+
64
+ lock.flock(File::LOCK_UN)
65
+ false
66
+ end
67
+
68
+ def let_go(tickets) = tickets.tap { @wait.call(tickets) }
69
+
70
+ def queue_dir(branch) = File.join(@root, digest(branch))
71
+ def digest(branch) = Digest::SHA256.hexdigest(branch)[0, 16]
72
+ end
73
+ end
74
+ end
@@ -21,7 +21,7 @@ module FunCi
21
21
 
22
22
  def cancel(new_commit_hash:)
23
23
  waited_before = (Time.now - WAITED_ON_SECONDS).utc.iso8601
24
- Persistence::ActiveRuns.cancellable(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
24
+ Persistence::ActiveRuns.cancel_stale(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
25
25
  .each { |run| cancel_run(run, new_commit_hash) }
26
26
  end
27
27
 
@@ -5,6 +5,8 @@ require_relative "../persistence/pipeline_recorder"
5
5
  require_relative "trigger_params"
6
6
  require_relative "trigger_command"
7
7
  require_relative "slot_run"
8
+ require_relative "slow_turns"
9
+ require_relative "worktrees"
8
10
  require_relative "stale_pipeline_canceller"
9
11
  require_relative "workspaces"
10
12
  require_relative "trunk_run"
@@ -96,10 +98,16 @@ module FunCi
96
98
 
97
99
  # After forking the slow suite the run holds a recorder of its own.
98
100
  def run_in(slot)
99
- slot_run = SlotRun.new(commit: @commit, io: @io, seams: @seams, slot: slot)
101
+ slot_run = SlotRun.new(commit: @commit, io: @io, seams: with_slow_turns, slot: slot)
100
102
  slot_run.run(config_for(slot)).tap { @seams = slot_run.seams }
101
103
  end
102
104
 
105
+ def with_slow_turns
106
+ return @seams if @seams.slow_turns
107
+
108
+ @seams.with(slow_turns: SlowTurns.new(Worktrees.new(@project).slow_turns_root))
109
+ end
110
+
103
111
  def cancel_stale_pipelines
104
112
  return unless recorder.db
105
113
 
@@ -34,9 +34,12 @@ module FunCi
34
34
  # `process_table` answers what ps lists, for a stage over budget; `trunk`
35
35
  # checks a commit against the trunk (#check(sha) answers a Trunk::Check,
36
36
  # or nil for no check), nil for the project's own; `priorities` what jobs
37
- # and the slow suite start under.
37
+ # and the slow suite start under; `slow_turns` what a slow suite waits on
38
+ # while an older one of its branch runs (#take(branch) yields once it is
39
+ # its turn), nil for the project's own SlowTurns.
38
40
  Seams = Data.define(:command_runner, :time_budgets, :commit_resolver, :recorder, :background_launcher, :workspace,
39
- :stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk, :priorities)
41
+ :stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk, :priorities,
42
+ :slow_turns)
40
43
 
41
44
  # Reopened rather than given as a block to Data.define, so tools that read
42
45
  # the source (mutineer) see these as Seams' methods.
@@ -52,7 +55,8 @@ module FunCi
52
55
  workspace: nil, commit_resolver: method(:full_sha), stage_dir: StageDir.method(:create),
53
56
  environment: ENV.to_h, clock: MONOTONIC,
54
57
  extractor_runner: Evidence::CommandRunner.method(:new),
55
- process_table: Evidence::ProcessTable.method(:now), trunk: nil, priorities: Priorities.for(RUBY_PLATFORM) }
58
+ process_table: Evidence::ProcessTable.method(:now), trunk: nil, priorities: Priorities.for(RUBY_PLATFORM),
59
+ slow_turns: nil }
56
60
  end
57
61
 
58
62
  # The full SHA of the commit `rev` names, or nil when the repository has none.
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require_relative "slot_lock"
5
+
6
+ module FunCi
7
+ module Pipeline
8
+ # A slow suite's place among its branch's: where its run's commit arrived
9
+ # (Persistence::Arrival), then its stage's id, so of two runs of one
10
+ # commit the later is the newer.
11
+ TurnTicket = Data.define(:place, :id) do
12
+ def name = format("%<place>020d-%<id>020d.lock", place: place, id: id)
13
+ end
14
+
15
+ # The tickets of a branch's slow suites waiting or running, as files in
16
+ # `dir`, each holding an flock on its own until its turn is over. A ticket
17
+ # appears already held (locked under a name of its own, then renamed), so
18
+ # one nobody holds was left by a suite that died, and goes.
19
+ class TurnQueue
20
+ # Blocks until the suite holding the ticket at each of `paths` is done.
21
+ WAIT = ->(paths) { paths.each { |path| TurnQueue.wait_on(path) } }
22
+
23
+ def self.wait_on(path)
24
+ File.open(path) { |file| file.flock(File::LOCK_SH) }
25
+ rescue Errno::ENOENT
26
+ nil
27
+ end
28
+
29
+ def initialize(dir)
30
+ @dir = dir
31
+ end
32
+
33
+ # Holds `ticket` in the queue until the block returns.
34
+ def holding(ticket)
35
+ path = File.join(@dir, ticket.name)
36
+ file = enter(path)
37
+ yield
38
+ ensure
39
+ FileUtils.rm_f(path) if file
40
+ file&.close
41
+ end
42
+
43
+ # The tickets held of suites newer than `ticket`'s, as a Set of paths.
44
+ def newer_than(ticket)
45
+ Dir.glob(File.join(@dir, "*.lock")).select { |path| live?(path) && File.basename(path) > ticket.name }.to_set
46
+ end
47
+
48
+ private
49
+
50
+ def enter(path)
51
+ FileUtils.mkdir_p(@dir)
52
+ locked("#{path}.#{Process.pid}.new").tap { |file| File.rename(file.path, path) }
53
+ end
54
+
55
+ def locked(path) = File.new(path, File::RDWR | File::CREAT, 0o644).tap { |file| file.flock(File::LOCK_EX) }
56
+
57
+ def live?(path)
58
+ return true if SlotLock.held?(path)
59
+
60
+ FileUtils.rm_f(path)
61
+ false
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require_relative "slot_lock"
5
+
6
+ module FunCi
7
+ module Pipeline
8
+ # The slots held only by a slow suite waiting its branch's turn
9
+ # (acceptance-tests.md, AT-9.19). Such a suite keeps its slot, whose
10
+ # checkout and build it runs in, and holds an flock on the slot's mark
11
+ # under `waiting/` beside the slots while it waits. WorktreePool hands out
12
+ # a further slot for each mark held, so a queued suite keeps no other
13
+ # branch's run waiting. A mark dies with the process that held it.
14
+ module WaitingSlots
15
+ DIR = "waiting"
16
+
17
+ # The mark of the slot at `slot_path`, held until it is closed.
18
+ def self.mark(slot_path)
19
+ path = mark_path(slot_path)
20
+ FileUtils.mkdir_p(File.dirname(path))
21
+ File.new(path, File::RDWR | File::CREAT, 0o644).tap { |mark| mark.flock(File::LOCK_EX) }
22
+ end
23
+
24
+ # How many slots under `root` a waiting slow suite holds now.
25
+ def self.count(root) = Dir.glob(File.join(root, DIR, "*.lock")).count { |path| SlotLock.held?(path) }
26
+
27
+ def self.mark_path(slot_path) = File.join(File.dirname(slot_path), DIR, "#{File.basename(slot_path)}.lock")
28
+ end
29
+ end
30
+ end
@@ -3,12 +3,14 @@
3
3
  require "fileutils"
4
4
  require_relative "slot"
5
5
  require_relative "worktrees"
6
+ require_relative "waiting_slots"
6
7
 
7
8
  module FunCi
8
9
  module Pipeline
9
10
  # Hands each pipeline a worktree slot of its own under
10
11
  # Worktrees#root, checked out at its commit, waiting
11
- # while every slot is taken.
12
+ # while every slot is taken. A slot held only by a slow suite waiting its
13
+ # turn takes no place among `size` (WaitingSlots).
12
14
  class WorktreePool
13
15
  DEFAULT_SIZE = 2
14
16
  # A slot nobody took for longer is cold: its caches are as warm as a
@@ -53,7 +55,8 @@ module FunCi
53
55
  def free_slot
54
56
  root = @worktrees.root
55
57
  FileUtils.mkdir_p(root)
56
- @size.times.lazy.filter_map { |index| try_lock(File.join(root, "slot-#{index}")) }.first
58
+ working = @size + WaitingSlots.count(root)
59
+ working.times.lazy.filter_map { |index| try_lock(File.join(root, "slot-#{index}")) }.first
57
60
  end
58
61
 
59
62
  # The lock stays open for as long as the slot is held, so no block.
@@ -25,6 +25,9 @@ module FunCi
25
25
  # Where each daily and weekly job has a worktree of its own (Jobs::Locks).
26
26
  def jobs_root = File.join(fun_ci_dir, "jobs")
27
27
 
28
+ # Where each branch's slow suites take turns (SlowTurns).
29
+ def slow_turns_root = File.join(fun_ci_dir, "slow")
30
+
28
31
  # Where fun-ci before 2.4 kept the worktrees: under the git directory.
29
32
  def legacy_dir = File.join(common_dir, "fun-ci")
30
33
 
@@ -50,7 +50,12 @@ module FunCi
50
50
  end
51
51
 
52
52
  # One check of a commit against one trunk tip (architecture.md, Checking against the trunk);
53
- # the tip is nil when there was none to check against.
54
- Check = Data.define(:commit, :tip, :merge)
53
+ # the tip is nil when there was none to check against. checked_at: when
54
+ # it was recorded, for a check read back; nil for one just made.
55
+ Check = Data.define(:commit, :tip, :merge, :checked_at)
56
+
57
+ class Check
58
+ def initialize(checked_at: nil, **) = super
59
+ end
55
60
  end
56
61
  end
@@ -2,5 +2,5 @@
2
2
 
3
3
  # Loaded on its own by the gemspec, so it must not require anything.
4
4
  module FunCi
5
- VERSION = "2.4.0"
5
+ VERSION = "2.5.0"
6
6
  end
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fun_ci
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.0
4
+ version: 2.5.0
5
5
  platform: x86_64-linux
6
6
  authors:
7
7
  - Erik Thyge Madsen
@@ -43,15 +43,17 @@ description: 'Opinionated local CI that checks your code before it leaves your m
43
43
  held to a strict time budget (30 s, 30 s, 10 s and 5 minutes). The pre-push hook
44
44
  stops a push whose lint, build or fast suite failed. There are two ways in. People
45
45
  watch fun-ci console, a terminal dashboard drawn in 24-bit colour by a Rust renderer
46
- (included in the Linux and macOS gems), where each milestone of a run plays its
47
- own scene: a rocket while it runs, fireworks when it passes, an explosion when it
48
- fails, and a quiet night scene when nothing has run for a while. Coding agents and
49
- scripts ask with commands that answer in exit codes: fun-ci wait blocks until a
50
- commit''s verdict is in, and fun-ci why prints what was kept about a failure (the
51
- failures a test runner reported, excerpts picked out by presets for 34 tools, log
52
- lines, the end of the output, with secrets masked), so nobody has to run a suite
53
- again to see what broke. Results are kept in SQLite. Any language works, since the
54
- stages are shell scripts; fun-ci init writes them for Ruby, Gradle and Maven projects.'
46
+ (included in the Linux and macOS gems), where each milestone of a run plays a scene
47
+ of its own: a rocket while it runs, gears when the build passes, fireworks or a
48
+ sunrise when the whole run does, an explosion when it fails, and a quiet night scene
49
+ when nothing has run for a while. Coding agents and scripts ask with commands that
50
+ answer in exit codes: fun-ci wait blocks until a commit''s verdict is in, and fun-ci
51
+ why prints what was kept about a failure (the failures a test runner reported, excerpts
52
+ picked out by presets for 34 tools, log lines, the end of the output, with secrets
53
+ masked), so nobody has to run a suite again to see what broke. Results are kept
54
+ in SQLite. Any language works, since the stages are shell scripts; fun-ci init writes
55
+ them for Ruby, Gradle, Maven, Rust, Go, Elixir, Dart, Swift, PHP, .NET, Python,
56
+ Deno, Bun, Node, Perl, and C and C++ built with CMake or make.'
55
57
  executables:
56
58
  - fun-ci
57
59
  extensions: []
@@ -84,6 +86,7 @@ files:
84
86
  - lib/fun_ci/agent/jobs_text.rb
85
87
  - lib/fun_ci/agent/live_pipeline.rb
86
88
  - lib/fun_ci/agent/next_step.rb
89
+ - lib/fun_ci/agent/onward.rb
87
90
  - lib/fun_ci/agent/options.rb
88
91
  - lib/fun_ci/agent/output.rb
89
92
  - lib/fun_ci/agent/report_stage.rb
@@ -117,20 +120,25 @@ files:
117
120
  - lib/fun_ci/cli_help.rb
118
121
  - lib/fun_ci/console/board_data.rb
119
122
  - lib/fun_ci/console/board_heartbeat.rb
123
+ - lib/fun_ci/console/branch_page.rb
120
124
  - lib/fun_ci/console/cancelled_folding.rb
121
125
  - lib/fun_ci/console/console_events.rb
122
126
  - lib/fun_ci/console/console_log.rb
123
127
  - lib/fun_ci/console/console_loop.rb
124
128
  - lib/fun_ci/console/console_session.rb
125
129
  - lib/fun_ci/console/console_state.rb
130
+ - lib/fun_ci/console/cursor.rb
131
+ - lib/fun_ci/console/cursor_rows.rb
126
132
  - lib/fun_ci/console/job_message.rb
127
133
  - lib/fun_ci/console/job_order.rb
134
+ - lib/fun_ci/console/job_row.rb
128
135
  - lib/fun_ci/console/job_rows.rb
129
136
  - lib/fun_ci/console/key_handler.rb
130
137
  - lib/fun_ci/console/launcher.rb
131
138
  - lib/fun_ci/console/milestone_events.rb
132
139
  - lib/fun_ci/console/milestones.rb
133
140
  - lib/fun_ci/console/paging.rb
141
+ - lib/fun_ci/console/question.rb
134
142
  - lib/fun_ci/console/renderer_lookup.rb
135
143
  - lib/fun_ci/console/renderer_process.rb
136
144
  - lib/fun_ci/console/row_order.rb
@@ -236,10 +244,14 @@ files:
236
244
  - lib/fun_ci/jobs/wall_clock_wait.rb
237
245
  - lib/fun_ci/persistence/active_jobs.rb
238
246
  - lib/fun_ci/persistence/active_runs.rb
247
+ - lib/fun_ci/persistence/arrival.rb
248
+ - lib/fun_ci/persistence/branch_visibility.rb
239
249
  - lib/fun_ci/persistence/database.rb
240
250
  - lib/fun_ci/persistence/died.rb
251
+ - lib/fun_ci/persistence/hidden_row_tables.rb
241
252
  - lib/fun_ci/persistence/job_recorder.rb
242
253
  - lib/fun_ci/persistence/job_runs.rb
254
+ - lib/fun_ci/persistence/job_visibility.rb
243
255
  - lib/fun_ci/persistence/null_recorder.rb
244
256
  - lib/fun_ci/persistence/output_tail.rb
245
257
  - lib/fun_ci/persistence/pipeline_recorder.rb
@@ -272,6 +284,7 @@ files:
272
284
  - lib/fun_ci/pipeline/slot.rb
273
285
  - lib/fun_ci/pipeline/slot_lock.rb
274
286
  - lib/fun_ci/pipeline/slot_run.rb
287
+ - lib/fun_ci/pipeline/slow_turns.rb
275
288
  - lib/fun_ci/pipeline/stage_dir.rb
276
289
  - lib/fun_ci/pipeline/stage_end.rb
277
290
  - lib/fun_ci/pipeline/stage_execution.rb
@@ -283,6 +296,8 @@ files:
283
296
  - lib/fun_ci/pipeline/trigger_command.rb
284
297
  - lib/fun_ci/pipeline/trigger_params.rb
285
298
  - lib/fun_ci/pipeline/trunk_run.rb
299
+ - lib/fun_ci/pipeline/turn_queue.rb
300
+ - lib/fun_ci/pipeline/waiting_slots.rb
286
301
  - lib/fun_ci/pipeline/workspaces.rb
287
302
  - lib/fun_ci/pipeline/worktree_pool.rb
288
303
  - lib/fun_ci/pipeline/worktree_prune.rb