fun_ci 2.1.1-aarch64-linux → 2.3.0-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +95 -0
- data/README.md +36 -6
- data/lib/fun_ci/agent/cancel_command.rb +52 -0
- data/lib/fun_ci/agent/command_support.rb +7 -0
- data/lib/fun_ci/agent/commands.rb +4 -1
- data/lib/fun_ci/agent/due_in.rb +21 -0
- data/lib/fun_ci/agent/events_command.rb +14 -7
- data/lib/fun_ci/agent/job_events.rb +56 -0
- data/lib/fun_ci/agent/job_json.rb +25 -0
- data/lib/fun_ci/agent/job_report.rb +89 -0
- data/lib/fun_ci/agent/job_why.rb +59 -0
- data/lib/fun_ci/agent/job_why_json.rb +31 -0
- data/lib/fun_ci/agent/job_why_text.rb +59 -0
- data/lib/fun_ci/agent/jobs_command.rb +43 -0
- data/lib/fun_ci/agent/jobs_text.rb +55 -0
- data/lib/fun_ci/agent/live_pipeline.rb +14 -6
- data/lib/fun_ci/agent/options.rb +3 -3
- data/lib/fun_ci/agent/output.rb +14 -2
- data/lib/fun_ci/agent/snapshots.rb +20 -1
- data/lib/fun_ci/agent/span.rb +24 -0
- data/lib/fun_ci/agent/stage_summary.rb +10 -6
- data/lib/fun_ci/agent/starts_in.rb +14 -0
- data/lib/fun_ci/agent/status_command.rb +3 -2
- data/lib/fun_ci/agent/status_json.rb +1 -1
- data/lib/fun_ci/agent/status_text.rb +32 -6
- data/lib/fun_ci/agent/trunk_reading.rb +2 -1
- data/lib/fun_ci/agent/wait_command.rb +2 -1
- data/lib/fun_ci/agent/why_command.rb +6 -2
- data/lib/fun_ci/cli.rb +1 -1
- data/lib/fun_ci/cli_help.rb +4 -1
- data/lib/fun_ci/console/board_data.rb +26 -13
- data/lib/fun_ci/console/console_state.rb +12 -6
- data/lib/fun_ci/console/job_message.rb +39 -0
- data/lib/fun_ci/console/job_order.rb +23 -0
- data/lib/fun_ci/console/job_rows.rb +21 -0
- data/lib/fun_ci/console/key_handler.rb +35 -14
- data/lib/fun_ci/console/paging.rb +26 -0
- data/lib/fun_ci/console/view.rb +18 -9
- data/lib/fun_ci/evidence/settings.rb +15 -8
- data/lib/fun_ci/evidence/settings_check.rb +14 -3
- data/lib/fun_ci/jobs/due.rb +34 -0
- data/lib/fun_ci/jobs/due_jobs.rb +50 -0
- data/lib/fun_ci/jobs/folders.rb +50 -0
- data/lib/fun_ci/jobs/job.rb +31 -0
- data/lib/fun_ci/jobs/job_fork.rb +52 -0
- data/lib/fun_ci/jobs/job_run.rb +104 -0
- data/lib/fun_ci/jobs/locks.rb +30 -0
- data/lib/fun_ci/jobs/schedule.rb +25 -0
- data/lib/fun_ci/jobs/site.rb +17 -0
- data/lib/fun_ci/jobs/standings.rb +50 -0
- data/lib/fun_ci/jobs/state.rb +27 -0
- data/lib/fun_ci/jobs/wall_clock_wait.rb +29 -0
- data/lib/fun_ci/persistence/active_jobs.rb +60 -0
- data/lib/fun_ci/persistence/database.rb +14 -4
- data/lib/fun_ci/persistence/job_recorder.rb +47 -0
- data/lib/fun_ci/persistence/job_runs.rb +80 -0
- data/lib/fun_ci/persistence/pipeline_run.rb +2 -1
- data/lib/fun_ci/persistence/raw_outputs.rb +3 -0
- data/lib/fun_ci/pipeline/budgets.rb +9 -0
- data/lib/fun_ci/pipeline/pipeline_forker.rb +42 -10
- data/lib/fun_ci/pipeline/priorities.rb +23 -0
- data/lib/fun_ci/pipeline/process_runner.rb +18 -6
- data/lib/fun_ci/pipeline/progress_reporter.rb +4 -0
- data/lib/fun_ci/pipeline/run_canceller.rb +24 -0
- data/lib/fun_ci/pipeline/slot.rb +12 -1
- data/lib/fun_ci/pipeline/slot_run.rb +19 -4
- data/lib/fun_ci/pipeline/stage_execution.rb +9 -4
- data/lib/fun_ci/pipeline/trigger.rb +17 -6
- data/lib/fun_ci/pipeline/trigger_command.rb +12 -2
- data/lib/fun_ci/pipeline/trigger_params.rb +20 -9
- data/lib/fun_ci/pipeline/worktree_pool.rb +2 -2
- data/lib/fun_ci/pipeline/worktree_prune.rb +8 -6
- data/lib/fun_ci/pipeline/worktrees.rb +11 -5
- data/lib/fun_ci/setup/agent_instructions.rb +24 -5
- data/lib/fun_ci/setup/commands.rb +3 -1
- data/lib/fun_ci/setup/hook_script.rb +9 -4
- data/lib/fun_ci/setup/hook_writer.rb +13 -5
- data/lib/fun_ci/setup/installed_hooks.rb +45 -0
- data/lib/fun_ci/setup/installer.rb +3 -2
- data/lib/fun_ci/setup/lfs_hook.rb +50 -0
- data/lib/fun_ci/setup/project_config.rb +13 -1
- data/lib/fun_ci/setup/settings.rb +40 -11
- data/lib/fun_ci/setup/setup_checker.rb +20 -3
- data/lib/fun_ci/trunk/shown.rb +7 -4
- data/lib/fun_ci/version.rb +1 -1
- data/libexec/fun-ci-renderer +0 -0
- metadata +36 -1
|
@@ -23,16 +23,26 @@ module FunCi
|
|
|
23
23
|
DRAIN_SECONDS = 1
|
|
24
24
|
|
|
25
25
|
# Where the command runs, what it finds in its environment besides
|
|
26
|
-
# fun-ci's own, the window what it prints is written to,
|
|
27
|
-
# with its pid when it runs over budget, before it is killed
|
|
28
|
-
|
|
26
|
+
# fun-ci's own, the window what it prints is written to, what to do
|
|
27
|
+
# with its pid when it runs over budget, before it is killed, the
|
|
28
|
+
# open files it holds as long as it runs, such as a lock, and what it
|
|
29
|
+
# is run under (a Priorities prefix, "" for fun-ci's own priority).
|
|
30
|
+
Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority)
|
|
29
31
|
|
|
30
32
|
class Launch
|
|
31
33
|
def initialize(**given)
|
|
32
|
-
super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil,
|
|
34
|
+
super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
|
|
35
|
+
**given)
|
|
33
36
|
end
|
|
34
37
|
end
|
|
35
38
|
|
|
39
|
+
# Where the first file a command holds is open in it: past the gate's 9.
|
|
40
|
+
FIRST_HELD = 10
|
|
41
|
+
|
|
42
|
+
# { descriptor in the command => file } for the files it holds. Not their
|
|
43
|
+
# own descriptors, which in a process with few files open may be the gate's.
|
|
44
|
+
def self.held_descriptors(files) = files.each_with_index.to_h { |file, index| [FIRST_HELD + index, file] }
|
|
45
|
+
|
|
36
46
|
# A command started and let go, and the pipe that carries what it prints.
|
|
37
47
|
Started = Data.define(:pid, :reader)
|
|
38
48
|
|
|
@@ -69,10 +79,12 @@ module FunCi
|
|
|
69
79
|
end
|
|
70
80
|
|
|
71
81
|
def spawn_gated(cmd, writer, gate, launch)
|
|
72
|
-
Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
|
|
73
|
-
out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir)
|
|
82
|
+
Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, "#{launch.priority}#{cmd}"),
|
|
83
|
+
out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir, **held(launch))
|
|
74
84
|
end
|
|
75
85
|
|
|
86
|
+
def held(launch) = ProcessRunner.held_descriptors(launch.held)
|
|
87
|
+
|
|
76
88
|
# The reader may be closed while this thread still reads, once a killed
|
|
77
89
|
# command's output has drained or stopped coming.
|
|
78
90
|
def read_until_closed(reader, printed)
|
|
@@ -17,6 +17,10 @@ module FunCi
|
|
|
17
17
|
@stdout.puts "fun-ci: fast #{result_marker(passed)}"
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
def cold_slot(seconds)
|
|
21
|
+
@stdout.puts "fun-ci: a new worktree, whose caches are empty, so lint and build have #{seconds}s this once."
|
|
22
|
+
end
|
|
23
|
+
|
|
20
24
|
def slow_launched
|
|
21
25
|
@stdout.puts "fun-ci: slow (running in background)"
|
|
22
26
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../persistence/active_runs"
|
|
4
|
+
require_relative "../persistence/active_jobs"
|
|
4
5
|
require_relative "../persistence/run_status"
|
|
5
6
|
require_relative "../persistence/write_trouble"
|
|
6
7
|
require_relative "slot_lock"
|
|
@@ -36,6 +37,26 @@ module FunCi
|
|
|
36
37
|
# The next poll looks again.
|
|
37
38
|
end
|
|
38
39
|
|
|
40
|
+
# Stops a daily or weekly job's processes and records it cancelled,
|
|
41
|
+
# unless it has finished meanwhile; a group its runner recorded while
|
|
42
|
+
# this looked is stopped once the runner is dead, as a run's late stages are.
|
|
43
|
+
def cancel_job(db, id)
|
|
44
|
+
Persistence::ActiveJobs.with_id(db, id).each do |job|
|
|
45
|
+
stop(job)
|
|
46
|
+
Persistence::ActiveJobs.with_id(db, id).each { |now| kill_late(job, now) }
|
|
47
|
+
Persistence::ActiveJobs.cancelled(db, id)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Records failed each daily or weekly job still running whose lock
|
|
52
|
+
# nobody holds: its process is gone (acceptance-tests.md, AT-13.6).
|
|
53
|
+
def record_dead_jobs(db)
|
|
54
|
+
Persistence::ActiveJobs.running(db).filter_map { |id, lock| id unless @slot_held.call(lock) }
|
|
55
|
+
.each { |id| Persistence::ActiveJobs.died(db, id) }
|
|
56
|
+
rescue SQLite3::BusyException, *Persistence::WriteTrouble::UNWRITABLE
|
|
57
|
+
# The next poll looks again.
|
|
58
|
+
end
|
|
59
|
+
|
|
39
60
|
def stop(run)
|
|
40
61
|
return unless alive?(run)
|
|
41
62
|
|
|
@@ -53,6 +74,9 @@ module FunCi
|
|
|
53
74
|
end
|
|
54
75
|
end
|
|
55
76
|
|
|
77
|
+
# The groups recorded since `before` was read.
|
|
78
|
+
def kill_late(before, now) = (now.stage_groups - before.stage_groups).each { |group| kill(-group) }
|
|
79
|
+
|
|
56
80
|
def alive?(run) = run.slot_lock.nil? || @slot_held.call(run.slot_lock)
|
|
57
81
|
|
|
58
82
|
# Whether a process with this pid exists; one this user may not signal does.
|
data/lib/fun_ci/pipeline/slot.rb
CHANGED
|
@@ -6,7 +6,7 @@ module FunCi
|
|
|
6
6
|
# file. The fast suite and the slow suite each hold it; it frees when the
|
|
7
7
|
# last of them lets go. A process that dies lets go with it.
|
|
8
8
|
class Slot
|
|
9
|
-
attr_reader :path, :lock_file
|
|
9
|
+
attr_reader :path, :lock_file, :lock
|
|
10
10
|
|
|
11
11
|
# +lock_file+ names the lock on disk; nil for a slot with nothing to lock.
|
|
12
12
|
def initialize(path, lock, lock_file = nil)
|
|
@@ -25,6 +25,17 @@ module FunCi
|
|
|
25
25
|
@holders -= 1
|
|
26
26
|
@lock.close if @holders.zero? && !@lock.closed?
|
|
27
27
|
end
|
|
28
|
+
|
|
29
|
+
# Whether its worktree was just made, so has no caches yet.
|
|
30
|
+
def cold? = false
|
|
31
|
+
|
|
32
|
+
# The same slot, before anyone shares it, in a worktree just made.
|
|
33
|
+
def as_cold = ColdSlot.new(path, lock, lock_file)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# A slot whose worktree was just made: its build starts from nothing.
|
|
37
|
+
class ColdSlot < Slot
|
|
38
|
+
def cold? = true
|
|
28
39
|
end
|
|
29
40
|
end
|
|
30
41
|
end
|
|
@@ -34,22 +34,36 @@ module FunCi
|
|
|
34
34
|
|
|
35
35
|
def recorder = @seams.recorder
|
|
36
36
|
def progress = ProgressReporter.new(stdout: @io.stdout)
|
|
37
|
-
|
|
37
|
+
|
|
38
|
+
def stage_runner(seams = @seams)
|
|
39
|
+
StageRunner.new(commit: @commit, stdout: @io.stdout, seams: seams, dir: @slot.path)
|
|
40
|
+
end
|
|
38
41
|
|
|
39
42
|
# This process's part is over; the slow suite may still be running.
|
|
40
43
|
def released(exit_code)
|
|
41
44
|
@slot.release
|
|
42
|
-
recorder.foreground_done
|
|
43
45
|
exit_code
|
|
44
46
|
end
|
|
45
47
|
|
|
46
48
|
def phase_one_passed?(config)
|
|
47
|
-
|
|
49
|
+
runner = stage_runner(phase_one_seams)
|
|
50
|
+
results = %w[lint build].to_h { |stage| [stage, Thread.new { runner.passes?(config, stage) }] }
|
|
48
51
|
.transform_values(&:value)
|
|
49
52
|
progress.phase_one_result(results)
|
|
50
53
|
results.values.all?
|
|
51
54
|
end
|
|
52
55
|
|
|
56
|
+
# A worktree just made has no caches, so its lint and build are no
|
|
57
|
+
# measure of the project's: they get the slow suite's budget, this once.
|
|
58
|
+
def phase_one_seams
|
|
59
|
+
return @seams unless @slot.cold?
|
|
60
|
+
|
|
61
|
+
cold = @seams.budgets["slow"]
|
|
62
|
+
progress.cold_slot(cold)
|
|
63
|
+
longer = %w[lint build].to_h { |stage| [stage, [@seams.budgets[stage], cold].max] }
|
|
64
|
+
@seams.with(time_budgets: @seams.time_budgets.merge(longer))
|
|
65
|
+
end
|
|
66
|
+
|
|
53
67
|
def fast_passed?(config)
|
|
54
68
|
passed = stage_runner.passes?(config, "fast")
|
|
55
69
|
progress.fast_result(passed)
|
|
@@ -65,7 +79,8 @@ module FunCi
|
|
|
65
79
|
# Called with the recorder and job to record the slow suite with, in
|
|
66
80
|
# whichever process runs it; the slot is let go once it is recorded.
|
|
67
81
|
def slow_suite(config, slot)
|
|
68
|
-
execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit
|
|
82
|
+
execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
|
|
83
|
+
launching: { priority: @seams.priorities.slow })
|
|
69
84
|
command = config.stage_command("slow", @commit.sha)
|
|
70
85
|
->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
|
|
71
86
|
end
|
|
@@ -7,12 +7,18 @@ module FunCi
|
|
|
7
7
|
module Pipeline
|
|
8
8
|
# Runs a stage that has been started, in `dir`, with a stage directory of
|
|
9
9
|
# its own, and records how it ended while it still holds both. The same
|
|
10
|
-
# for every stage, the slow suite in its forked child included
|
|
10
|
+
# for every stage, the slow suite in its forked child included, and for
|
|
11
|
+
# a daily or weekly job.
|
|
11
12
|
class StageExecution
|
|
12
|
-
|
|
13
|
+
# launching: what the script is launched with besides its output window
|
|
14
|
+
# (ProcessRunner::Launch): what it is told about itself, by default its
|
|
15
|
+
# stage's name in FUN_CI_STAGE (acceptance-tests.md, AT-10.7), and the
|
|
16
|
+
# files it holds open.
|
|
17
|
+
def initialize(seams:, dir:, commit:, launching: {})
|
|
13
18
|
@seams = seams
|
|
14
19
|
@dir = dir
|
|
15
20
|
@commit = commit
|
|
21
|
+
@launching = launching
|
|
16
22
|
end
|
|
17
23
|
|
|
18
24
|
# Answers [the outcome recorded, what the stage printed].
|
|
@@ -36,10 +42,9 @@ module FunCi
|
|
|
36
42
|
StageEnd::Finished.new(output: output, status: status, timed_out: timed_out, overrun: overrun)
|
|
37
43
|
end
|
|
38
44
|
|
|
39
|
-
# The stage learns its name from FUN_CI_STAGE (acceptance-tests.md, AT-10.7).
|
|
40
45
|
def execute(stage, command, stage_dir, before_kill, &)
|
|
41
46
|
window = stage_dir.window
|
|
42
|
-
launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, output: window,
|
|
47
|
+
launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, **@launching, output: window,
|
|
43
48
|
before_kill: before_kill)
|
|
44
49
|
@seams.executor(@dir).call(command, @seams.budgets[stage], launch, &)
|
|
45
50
|
ensure
|
|
@@ -29,9 +29,12 @@ module FunCi
|
|
|
29
29
|
def run
|
|
30
30
|
config = Setup::ProjectConfig.new(@project)
|
|
31
31
|
return handle_config_errors(config) if config.validate.any?
|
|
32
|
-
return unknown_commit unless known_commit?
|
|
33
32
|
|
|
34
|
-
|
|
33
|
+
full = full_sha
|
|
34
|
+
return unknown_commit unless full
|
|
35
|
+
|
|
36
|
+
@commit = @commit.with(sha: full)
|
|
37
|
+
start_run(config)
|
|
35
38
|
checking_the_trunk { run_in(workspace.acquire(@commit.sha)) }
|
|
36
39
|
end
|
|
37
40
|
|
|
@@ -44,7 +47,8 @@ module FunCi
|
|
|
44
47
|
private
|
|
45
48
|
|
|
46
49
|
def recorder = @seams.recorder
|
|
47
|
-
|
|
50
|
+
# Runs are kept under the full SHA, whatever revision named the commit.
|
|
51
|
+
def full_sha = @commit.sha == NULL_SHA ? NULL_SHA : @seams.commit_resolver.call(@commit.sha)
|
|
48
52
|
|
|
49
53
|
def workspace = @seams.workspace || Workspaces.for(@project, @commit.sha)
|
|
50
54
|
def branch = Persistence::Branch.new(project: @project, name: @commit.branch)
|
|
@@ -56,11 +60,16 @@ module FunCi
|
|
|
56
60
|
committed.folder_exists? ? committed : Setup::ProjectConfig.new(@project)
|
|
57
61
|
end
|
|
58
62
|
|
|
59
|
-
# The check is recorded by the recorder the run holds once its stages are
|
|
63
|
+
# The check is recorded by the recorder the run holds once its stages are
|
|
64
|
+
# done, and before the run says its foreground is: until then, a reader
|
|
65
|
+
# takes a check not yet recorded to be still going.
|
|
60
66
|
def checking_the_trunk
|
|
61
67
|
trunk = TrunkRun.start(@seams.trunk || Trunk::Checker.for(@project), @commit.sha, recorder)
|
|
62
68
|
@io.stdout.puts(trunk.notice) if trunk.notice
|
|
63
|
-
yield.tap
|
|
69
|
+
yield.tap do
|
|
70
|
+
trunk.finish(recorder)
|
|
71
|
+
recorder.foreground_done
|
|
72
|
+
end
|
|
64
73
|
end
|
|
65
74
|
|
|
66
75
|
def handle_config_errors(config)
|
|
@@ -77,7 +86,9 @@ module FunCi
|
|
|
77
86
|
1
|
|
78
87
|
end
|
|
79
88
|
|
|
80
|
-
|
|
89
|
+
# A mistake in the config is named, and its defaults used.
|
|
90
|
+
def start_run(config)
|
|
91
|
+
config.settings_errors.each { |e| @io.stdout.puts "fun-ci: #{e}" }
|
|
81
92
|
recorder.report_trouble_to(@io.stdout)
|
|
82
93
|
cancel_stale_pipelines
|
|
83
94
|
recorder.create_run(commit_hash: @commit.sha, branch: @commit.branch, project_path: @project)
|
|
@@ -56,11 +56,21 @@ module FunCi
|
|
|
56
56
|
return 0 unless forked
|
|
57
57
|
|
|
58
58
|
say_how_to_wait(commit.sha[0, 7])
|
|
59
|
-
|
|
59
|
+
say(forked)
|
|
60
60
|
0
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
|
|
63
|
+
# The config's mistakes and the first fetch's notice, and on stderr, why
|
|
64
|
+
# the jobs didn't start; the run itself prints nowhere.
|
|
65
|
+
def say(forked)
|
|
66
|
+
Setup::ProjectConfig.new(@project).settings_errors.each { |e| @io.stdout.puts "fun-ci: #{e}" }
|
|
67
|
+
@io.stdout.puts(forked.notice) if forked.notice
|
|
68
|
+
@io.stderr.puts(forked.jobs) if forked.jobs
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def say_how_to_wait(sha)
|
|
72
|
+
@io.stdout.puts("fun-ci: testing #{sha}. Verdict: fun-ci wait #{sha} --need all --follow-branch")
|
|
73
|
+
end
|
|
64
74
|
end
|
|
65
75
|
end
|
|
66
76
|
end
|
|
@@ -5,13 +5,13 @@ require_relative "../persistence/pipeline_recorder"
|
|
|
5
5
|
require_relative "command_executor"
|
|
6
6
|
require_relative "stage_dir"
|
|
7
7
|
require_relative "git_environment"
|
|
8
|
+
require_relative "priorities"
|
|
9
|
+
require_relative "budgets"
|
|
8
10
|
require_relative "../evidence/command_runner"
|
|
9
11
|
require_relative "../evidence/process_table"
|
|
10
12
|
|
|
11
13
|
module FunCi
|
|
12
14
|
module Pipeline
|
|
13
|
-
DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
|
|
14
|
-
|
|
15
15
|
Commit = Data.define(:sha, :branch)
|
|
16
16
|
|
|
17
17
|
# On a detached HEAD git names no branch, and the hooks pass "": the run
|
|
@@ -33,22 +33,33 @@ module FunCi
|
|
|
33
33
|
# own extractors (`run:` entries), given dir:, env: and scratch:;
|
|
34
34
|
# `process_table` answers what ps lists, for a stage over budget; `trunk`
|
|
35
35
|
# checks a commit against the trunk (#check(sha) answers a Trunk::Check,
|
|
36
|
-
# or nil for no check), nil for the project's own
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
# or nil for no check), nil for the project's own; `priorities` what jobs
|
|
37
|
+
# and the slow suite start under.
|
|
38
|
+
Seams = Data.define(:command_runner, :time_budgets, :commit_resolver, :recorder, :background_launcher, :workspace,
|
|
39
|
+
:stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk, :priorities)
|
|
39
40
|
|
|
40
41
|
# Reopened rather than given as a block to Data.define, so tools that read
|
|
41
42
|
# the source (mutineer) see these as Seams' methods.
|
|
42
43
|
class Seams
|
|
44
|
+
# Seconds, for the evidence budget. Kept out of `defaults`' hash: a block
|
|
45
|
+
# on a line the hash continues onto gives that line the block's count,
|
|
46
|
+
# and the mutation lane then runs the hash's later lines' mutants only
|
|
47
|
+
# against tests that call it (test/support/statement_coverage.rb).
|
|
48
|
+
MONOTONIC = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
49
|
+
|
|
43
50
|
def self.defaults
|
|
44
51
|
{ command_runner: nil, time_budgets: {}, recorder: Persistence::NullRecorder.new, background_launcher: nil,
|
|
45
|
-
workspace: nil,
|
|
46
|
-
environment: ENV.to_h, clock:
|
|
52
|
+
workspace: nil, commit_resolver: method(:full_sha), stage_dir: StageDir.method(:create),
|
|
53
|
+
environment: ENV.to_h, clock: MONOTONIC,
|
|
47
54
|
extractor_runner: Evidence::CommandRunner.method(:new),
|
|
48
|
-
process_table: Evidence::ProcessTable.method(:now), trunk: nil }
|
|
55
|
+
process_table: Evidence::ProcessTable.method(:now), trunk: nil, priorities: Priorities.for(RUBY_PLATFORM) }
|
|
49
56
|
end
|
|
50
57
|
|
|
51
|
-
|
|
58
|
+
# The full SHA of the commit `rev` names, or nil when the repository has none.
|
|
59
|
+
def self.full_sha(rev)
|
|
60
|
+
out, status = Open3.capture2(GitEnvironment::CLEAN, "git", "rev-parse", "--verify", "-q", "#{rev}^{commit}")
|
|
61
|
+
out.strip if status.success?
|
|
62
|
+
end
|
|
52
63
|
|
|
53
64
|
def initialize(**given) = super(**self.class.defaults.merge(given))
|
|
54
65
|
def budgets = DEFAULT_BUDGETS.merge(time_budgets)
|
|
@@ -20,10 +20,10 @@ module FunCi
|
|
|
20
20
|
@waiter = waiter
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# A ColdSlot when the slot's worktree had to be made.
|
|
23
24
|
def acquire(sha)
|
|
24
25
|
slot = wait_for_free_slot
|
|
25
|
-
@worktrees.check_out(slot.path, sha)
|
|
26
|
-
slot
|
|
26
|
+
@worktrees.check_out(slot.path, sha) ? slot.as_cold : slot
|
|
27
27
|
rescue StandardError
|
|
28
28
|
slot&.release
|
|
29
29
|
raise
|
|
@@ -5,8 +5,9 @@ require_relative "worktrees"
|
|
|
5
5
|
|
|
6
6
|
module FunCi
|
|
7
7
|
module Pipeline
|
|
8
|
-
# Removes every slot of a project's worktree pool
|
|
9
|
-
#
|
|
8
|
+
# Removes every slot of a project's worktree pool and every daily or
|
|
9
|
+
# weekly job's worktree, and git's entries for them, when no run or job
|
|
10
|
+
# holds one. It takes each slot's lock first, so a run
|
|
10
11
|
# starting meanwhile waits until the slot is gone and then checks it out
|
|
11
12
|
# afresh.
|
|
12
13
|
class WorktreePrune
|
|
@@ -16,8 +17,8 @@ module FunCi
|
|
|
16
17
|
@worktrees = worktrees
|
|
17
18
|
end
|
|
18
19
|
|
|
19
|
-
# How many
|
|
20
|
-
#
|
|
20
|
+
# How many worktrees it removed. Raises Busy, removing nothing, while a
|
|
21
|
+
# run or a job holds one.
|
|
21
22
|
def run
|
|
22
23
|
locks = take_locks
|
|
23
24
|
locks.each { |lock| FileUtils.rm_rf(lock.path.delete_suffix(".lock")) }
|
|
@@ -37,12 +38,13 @@ module FunCi
|
|
|
37
38
|
next if locks.last.flock(File::LOCK_EX | File::LOCK_NB)
|
|
38
39
|
|
|
39
40
|
locks.each(&:close)
|
|
40
|
-
raise Busy, "
|
|
41
|
+
raise Busy, "fun-ci is running something in #{path}"
|
|
41
42
|
end
|
|
42
43
|
end
|
|
43
44
|
|
|
44
45
|
def slot_paths
|
|
45
|
-
Dir.glob(File.join(@worktrees.root, "slot-*"))
|
|
46
|
+
(Dir.glob(File.join(@worktrees.root, "slot-*")) + Dir.glob(File.join(@worktrees.jobs_root, "*")))
|
|
47
|
+
.map { |path| path.delete_suffix(".lock") }.uniq
|
|
46
48
|
end
|
|
47
49
|
end
|
|
48
50
|
end
|
|
@@ -13,18 +13,20 @@ module FunCi
|
|
|
13
13
|
@project_root = project_root
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
def root
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
def root = File.join(fun_ci_dir, "worktrees")
|
|
17
|
+
|
|
18
|
+
# Where each daily and weekly job has a worktree of its own (Jobs::Locks).
|
|
19
|
+
def jobs_root = File.join(fun_ci_dir, "jobs")
|
|
20
20
|
|
|
21
21
|
# Leaves +path+ as a clean detached checkout of +sha+. Ignored files stay,
|
|
22
22
|
# so caches such as vendor/bundle survive between runs in the same slot.
|
|
23
|
+
# Answers whether it made the worktree, which then has no caches.
|
|
23
24
|
def check_out(path, sha)
|
|
24
|
-
return add(path, sha) unless File.exist?(File.join(path, ".git"))
|
|
25
|
+
return add(path, sha).then { true } unless File.exist?(File.join(path, ".git"))
|
|
25
26
|
|
|
26
27
|
git(path, "checkout", "--detach", "--force", sha)
|
|
27
28
|
git(path, "clean", "-fd")
|
|
29
|
+
false
|
|
28
30
|
end
|
|
29
31
|
|
|
30
32
|
# Has git forget the worktrees whose directories are gone.
|
|
@@ -32,6 +34,10 @@ module FunCi
|
|
|
32
34
|
|
|
33
35
|
private
|
|
34
36
|
|
|
37
|
+
def fun_ci_dir
|
|
38
|
+
File.join(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root), "fun-ci")
|
|
39
|
+
end
|
|
40
|
+
|
|
35
41
|
def add(path, sha) = git(@project_root, "worktree", "add", "--detach", "--force", path, sha)
|
|
36
42
|
|
|
37
43
|
def git(dir, *args)
|
|
@@ -4,7 +4,8 @@ module FunCi
|
|
|
4
4
|
module Setup
|
|
5
5
|
# What `fun-ci init` tells a coding agent working in the project, in the
|
|
6
6
|
# instructions file agents read (acceptance-tests.md, AT-9.15): AGENTS.md,
|
|
7
|
-
# or CLAUDE.md when that is the only one. The marker keeps it to once
|
|
7
|
+
# or CLAUDE.md when that is the only one. The marker keeps it to once, and
|
|
8
|
+
# finds a section an older fun-ci wrote, to bring it up to date.
|
|
8
9
|
module AgentInstructions
|
|
9
10
|
MARKER = "<!-- fun-ci: agent instructions -->"
|
|
10
11
|
SECTION = <<~MD.freeze
|
|
@@ -13,22 +14,40 @@ module FunCi
|
|
|
13
14
|
#{MARKER}
|
|
14
15
|
This project runs fun-ci on every commit. After each commit, run the
|
|
15
16
|
`fun-ci wait` command it prints, in the background, and act on its exit
|
|
16
|
-
code: 0 passed, 1 failed, 2 over budget, 3 undecided
|
|
17
|
+
code: 0 passed, 1 failed, 2 over budget, 3 undecided. A newer commit on
|
|
18
|
+
the branch cancels the run and the wait moves on to the newer one's.
|
|
17
19
|
`fun-ci wait --need all` must exit 0 before the work is called done.
|
|
18
20
|
Before calling work done, run `fun-ci wait --need all --trunk`; 6 means
|
|
19
21
|
the commit conflicts with the trunk, which has to be integrated first,
|
|
20
22
|
as it prints.
|
|
21
23
|
`fun-ci runs` lists recent runs, and `fun-ci status` says where one stands.
|
|
24
|
+
`fun-ci jobs` lists the daily and weekly jobs, which run beside the
|
|
25
|
+
pipeline and never hold up a verdict, and `fun-ci why --job NAME` says
|
|
26
|
+
why one failed.
|
|
22
27
|
MD
|
|
23
28
|
|
|
29
|
+
# A section in the text: from the heading over the marker, if it has
|
|
30
|
+
# one, to the next heading or the end.
|
|
31
|
+
WRITTEN = /^(?:## fun-ci\n\s*)?#{Regexp.escape(MARKER)}\n.*?(?=^#|\z)/m
|
|
32
|
+
|
|
24
33
|
def self.file_for(names) = names.include?("CLAUDE.md") && !names.include?("AGENTS.md") ? "CLAUDE.md" : "AGENTS.md"
|
|
25
34
|
|
|
26
|
-
# The text with the section added, or nil when it
|
|
35
|
+
# The text with the section added, or brought up to date; nil when it
|
|
36
|
+
# has this one already.
|
|
27
37
|
def self.merged(text)
|
|
28
|
-
|
|
38
|
+
merged = text.include?(MARKER) ? replaced(text) : added(text)
|
|
39
|
+
merged unless merged == text
|
|
40
|
+
end
|
|
29
41
|
|
|
30
|
-
|
|
42
|
+
def self.said(text, file)
|
|
43
|
+
return "Told agents what to do after a commit, in #{file}." unless text.include?(MARKER)
|
|
44
|
+
|
|
45
|
+
"Brought the fun-ci section of #{file} up to date."
|
|
31
46
|
end
|
|
47
|
+
|
|
48
|
+
def self.added(text) = text.strip.empty? ? SECTION : "#{text.rstrip}\n\n#{SECTION}"
|
|
49
|
+
def self.replaced(text) = text.sub(WRITTEN) { Regexp.last_match.post_match.empty? ? SECTION : "#{SECTION}\n" }
|
|
50
|
+
private_class_method :added, :replaced
|
|
32
51
|
end
|
|
33
52
|
end
|
|
34
53
|
end
|
|
@@ -32,7 +32,9 @@ module FunCi
|
|
|
32
32
|
install(args.any? ? [args.first] : HookScript.types)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
def check(_args)
|
|
35
|
+
def check(_args)
|
|
36
|
+
SetupChecker.run(project_root: @project_root, stdout: @stdout, hooks_dir: @hooks_dir.call(@project_root))
|
|
37
|
+
end
|
|
36
38
|
|
|
37
39
|
private
|
|
38
40
|
|
|
@@ -7,11 +7,15 @@ module FunCi
|
|
|
7
7
|
MARKER = "# fun-ci-managed-hook"
|
|
8
8
|
NULL_SHA = "0" * 40
|
|
9
9
|
|
|
10
|
+
NOT_INSTALLED = %(echo "fun-ci is not installed or not on PATH, so this runs without CI. ) +
|
|
11
|
+
%(Install it with: gem install fun_ci" >&2)
|
|
12
|
+
INSTALLED = "command -v fun-ci >/dev/null 2>&1"
|
|
13
|
+
|
|
10
14
|
GUARD = <<~SH.freeze
|
|
11
15
|
#!/bin/sh
|
|
12
16
|
#{MARKER}
|
|
13
|
-
if !
|
|
14
|
-
|
|
17
|
+
if ! #{INSTALLED}; then
|
|
18
|
+
#{NOT_INSTALLED}
|
|
15
19
|
exit 0
|
|
16
20
|
fi
|
|
17
21
|
SH
|
|
@@ -26,7 +30,7 @@ module FunCi
|
|
|
26
30
|
# Before a push: the fast verdict of each commit git says it pushes
|
|
27
31
|
# (acceptance-tests.md, AT-9.13). A deleted ref pushes the null SHA, and
|
|
28
32
|
# exit 5 is a project not set up for fun-ci, which pushes without CI.
|
|
29
|
-
|
|
33
|
+
PRE_PUSH_WAITS = <<~SH.freeze
|
|
30
34
|
status=0
|
|
31
35
|
while read -r local_ref local_sha remote_ref remote_sha; do
|
|
32
36
|
[ "$local_sha" = "#{NULL_SHA}" ] && continue
|
|
@@ -34,9 +38,10 @@ module FunCi
|
|
|
34
38
|
code=$?
|
|
35
39
|
[ "$code" -eq 0 ] || [ "$code" -eq 5 ] || status=1
|
|
36
40
|
done
|
|
37
|
-
exit $status
|
|
38
41
|
SH
|
|
39
42
|
|
|
43
|
+
PRE_PUSH = "#{PRE_PUSH_WAITS}exit $status\n".freeze
|
|
44
|
+
|
|
40
45
|
BODIES = { "post-commit" => POST_COMMIT, "pre-push" => PRE_PUSH }.freeze
|
|
41
46
|
|
|
42
47
|
def self.types = BODIES.keys
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
require_relative "hook_script"
|
|
5
|
+
require_relative "lfs_hook"
|
|
5
6
|
|
|
6
7
|
module FunCi
|
|
7
8
|
module Setup
|
|
@@ -23,8 +24,9 @@ module FunCi
|
|
|
23
24
|
return reject("Unknown hook type: #{@hook_type}") unless HookScript.types.include?(@hook_type)
|
|
24
25
|
return skip("Hook #{@hook_type} already exists from another tool, so it was left alone.") if foreign_hook?
|
|
25
26
|
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
lfs = lfs?
|
|
28
|
+
write_hook(lfs ? LfsHook.script(@hook_type) : HookScript.for(@hook_type))
|
|
29
|
+
@stdout.puts "Installed #{@hook_type} hook#{", which runs git-lfs's too" if lfs}."
|
|
28
30
|
0
|
|
29
31
|
end
|
|
30
32
|
|
|
@@ -33,13 +35,19 @@ module FunCi
|
|
|
33
35
|
def git_repo? = !@hooks_dir.nil?
|
|
34
36
|
def hook_path = File.join(@hooks_dir, @hook_type)
|
|
35
37
|
|
|
38
|
+
def existing = File.exist?(hook_path) ? File.read(hook_path) : ""
|
|
39
|
+
|
|
40
|
+
# git-lfs's own hook is not left alone: fun-ci's runs it.
|
|
36
41
|
def foreign_hook?
|
|
37
|
-
|
|
42
|
+
script = existing
|
|
43
|
+
!script.empty? && !HookScript.managed?(script) && !LfsHook.stock?(script, @hook_type)
|
|
38
44
|
end
|
|
39
45
|
|
|
40
|
-
def
|
|
46
|
+
def lfs? = LfsHook.runs_lfs?(existing, @hook_type)
|
|
47
|
+
|
|
48
|
+
def write_hook(script)
|
|
41
49
|
FileUtils.mkdir_p(@hooks_dir)
|
|
42
|
-
File.write(hook_path,
|
|
50
|
+
File.write(hook_path, script)
|
|
43
51
|
File.chmod(0o755, hook_path)
|
|
44
52
|
end
|
|
45
53
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lfs_hook"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Setup
|
|
7
|
+
# The post-commit and pre-push hooks git runs, as `fun-ci check` warns of
|
|
8
|
+
# them: one that doesn't run fun-ci tests no commit, or lets a push
|
|
9
|
+
# through without a verdict. `fun-ci install-hooks` writes either when it
|
|
10
|
+
# is missing or git-lfs's own; another tool's has to call fun-ci itself.
|
|
11
|
+
class InstalledHooks
|
|
12
|
+
WHAT = { "post-commit" => "No commit is tested", "pre-push" => "No push waits for a verdict" }.freeze
|
|
13
|
+
INSTALL = "Run `fun-ci install-hooks`"
|
|
14
|
+
CALL = { "post-commit" => "`fun-ci trigger --background <commit> <branch>`",
|
|
15
|
+
"pre-push" => "`fun-ci wait <commit> --need fast` for each commit pushed" }.freeze
|
|
16
|
+
|
|
17
|
+
# hooks_dir: where git runs the project's hooks from; nil outside a git repository.
|
|
18
|
+
def initialize(hooks_dir)
|
|
19
|
+
@hooks_dir = hooks_dir
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def warnings
|
|
23
|
+
return [] unless @hooks_dir
|
|
24
|
+
|
|
25
|
+
WHAT.keys.map { |type| warning(type, script(type)) }.compact
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def script(type)
|
|
31
|
+
path = File.join(@hooks_dir, type)
|
|
32
|
+
File.exist?(path) ? File.read(path) : ""
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def warning(type, script)
|
|
36
|
+
return nil if script.include?("fun-ci")
|
|
37
|
+
return "#{WHAT[type]}: the #{type} hook doesn't run fun-ci. #{INSTALL}" if install_fixes?(script, type)
|
|
38
|
+
|
|
39
|
+
"#{WHAT[type]}: the #{type} hook is another tool's and doesn't run fun-ci. Have it run #{CALL[type]}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def install_fixes?(script, type) = script.empty? || LfsHook.stock?(script, type)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -44,11 +44,12 @@ module FunCi
|
|
|
44
44
|
def tell_agents
|
|
45
45
|
file = AgentInstructions.file_for(Dir.children(@project_root))
|
|
46
46
|
path = File.join(@project_root, file)
|
|
47
|
-
|
|
47
|
+
told = File.exist?(path) ? File.read(path) : ""
|
|
48
|
+
merged = AgentInstructions.merged(told)
|
|
48
49
|
return unless merged
|
|
49
50
|
|
|
50
51
|
File.write(path, merged)
|
|
51
|
-
@stdout.puts
|
|
52
|
+
@stdout.puts AgentInstructions.said(told, file)
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
def initialised?
|