fun_ci 2.3.0-x86_64-linux → 2.5.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +143 -0
- data/README.md +20 -10
- data/lib/fun_ci/agent/events.rb +15 -7
- data/lib/fun_ci/agent/job_events.rb +12 -5
- data/lib/fun_ci/agent/onward.rb +21 -0
- data/lib/fun_ci/agent/output.rb +7 -0
- data/lib/fun_ci/agent/reports.rb +3 -0
- data/lib/fun_ci/agent/snapshots.rb +25 -7
- data/lib/fun_ci/agent/status_text.rb +9 -5
- data/lib/fun_ci/agent/wait_command.rb +37 -5
- data/lib/fun_ci/cli.rb +2 -0
- data/lib/fun_ci/cli_command_help.rb +51 -0
- data/lib/fun_ci/cli_help.rb +9 -9
- data/lib/fun_ci/console/board_data.rb +50 -10
- data/lib/fun_ci/console/branch_page.rb +9 -0
- data/lib/fun_ci/console/console_state.rb +22 -8
- data/lib/fun_ci/console/cursor.rb +22 -0
- data/lib/fun_ci/console/cursor_rows.rb +37 -0
- data/lib/fun_ci/console/job_message.rb +5 -4
- data/lib/fun_ci/console/job_order.rb +5 -3
- data/lib/fun_ci/console/job_row.rb +17 -0
- data/lib/fun_ci/console/job_rows.rb +14 -3
- data/lib/fun_ci/console/key_handler.rb +88 -51
- data/lib/fun_ci/console/question.rb +63 -0
- data/lib/fun_ci/console/run_message.rb +3 -3
- data/lib/fun_ci/console/trunk_marks.rb +3 -1
- data/lib/fun_ci/console/view.rb +20 -6
- data/lib/fun_ci/evidence/about_stage.rb +1 -1
- data/lib/fun_ci/evidence/collector.rb +6 -3
- data/lib/fun_ci/evidence/outcome.rb +6 -3
- data/lib/fun_ci/jobs/job_run.rb +10 -9
- data/lib/fun_ci/jobs/locks.rb +1 -1
- data/lib/fun_ci/persistence/active_jobs.rb +2 -11
- data/lib/fun_ci/persistence/active_runs.rb +48 -16
- data/lib/fun_ci/persistence/arrival.rb +18 -0
- data/lib/fun_ci/persistence/branch_visibility.rb +40 -0
- data/lib/fun_ci/persistence/database.rb +28 -2
- data/lib/fun_ci/persistence/died.rb +21 -0
- data/lib/fun_ci/persistence/hidden_row_tables.rb +24 -0
- data/lib/fun_ci/persistence/job_recorder.rb +1 -0
- data/lib/fun_ci/persistence/job_runs.rb +17 -7
- data/lib/fun_ci/persistence/job_visibility.rb +43 -0
- data/lib/fun_ci/persistence/null_recorder.rb +4 -0
- data/lib/fun_ci/persistence/pipeline_recorder.rb +23 -3
- data/lib/fun_ci/persistence/pipeline_run.rb +25 -11
- data/lib/fun_ci/persistence/stage_job.rb +25 -0
- data/lib/fun_ci/persistence/trunk_checks.rb +7 -6
- data/lib/fun_ci/pipeline/group_kill.rb +38 -0
- data/lib/fun_ci/pipeline/pipeline_forker.rb +3 -1
- data/lib/fun_ci/pipeline/process_runner.rb +14 -8
- data/lib/fun_ci/pipeline/process_tree.rb +37 -0
- data/lib/fun_ci/pipeline/progress_reporter.rb +2 -1
- data/lib/fun_ci/pipeline/run_canceller.rb +16 -6
- data/lib/fun_ci/pipeline/slot.rb +12 -0
- data/lib/fun_ci/pipeline/slot_run.rb +20 -5
- data/lib/fun_ci/pipeline/slow_turns.rb +74 -0
- data/lib/fun_ci/pipeline/stage_end.rb +2 -1
- data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb +1 -1
- data/lib/fun_ci/pipeline/trigger.rb +9 -1
- data/lib/fun_ci/pipeline/trigger_params.rb +7 -3
- data/lib/fun_ci/pipeline/turn_queue.rb +65 -0
- data/lib/fun_ci/pipeline/waiting_slots.rb +30 -0
- data/lib/fun_ci/pipeline/workspaces.rb +2 -1
- data/lib/fun_ci/pipeline/worktree_pool.rb +27 -7
- data/lib/fun_ci/pipeline/worktree_prune.rb +9 -4
- data/lib/fun_ci/pipeline/worktrees.rb +41 -7
- data/lib/fun_ci/setup/commands.rb +16 -7
- data/lib/fun_ci/setup/eslint_detection.rb +38 -0
- data/lib/fun_ci/setup/init_options.rb +34 -0
- data/lib/fun_ci/setup/installer.rb +41 -7
- data/lib/fun_ci/setup/placeholders.rb +30 -0
- data/lib/fun_ci/setup/project_config.rb +1 -0
- data/lib/fun_ci/setup/settings.rb +21 -1
- data/lib/fun_ci/setup/setup_checker.rb +5 -3
- data/lib/fun_ci/setup/stage_templates/javascript.rb +14 -4
- data/lib/fun_ci/setup/stage_templates/placeholders.rb +21 -0
- data/lib/fun_ci/setup/stage_templates.rb +6 -4
- data/lib/fun_ci/setup/template_writer.rb +3 -5
- data/lib/fun_ci/trunk/check.rb +7 -2
- data/lib/fun_ci/version.rb +1 -1
- data/libexec/fun-ci-renderer +0 -0
- metadata +33 -10
|
@@ -5,6 +5,8 @@ require_relative "../persistence/pipeline_recorder"
|
|
|
5
5
|
require_relative "trigger_params"
|
|
6
6
|
require_relative "trigger_command"
|
|
7
7
|
require_relative "slot_run"
|
|
8
|
+
require_relative "slow_turns"
|
|
9
|
+
require_relative "worktrees"
|
|
8
10
|
require_relative "stale_pipeline_canceller"
|
|
9
11
|
require_relative "workspaces"
|
|
10
12
|
require_relative "trunk_run"
|
|
@@ -96,10 +98,16 @@ module FunCi
|
|
|
96
98
|
|
|
97
99
|
# After forking the slow suite the run holds a recorder of its own.
|
|
98
100
|
def run_in(slot)
|
|
99
|
-
slot_run = SlotRun.new(commit: @commit, io: @io, seams:
|
|
101
|
+
slot_run = SlotRun.new(commit: @commit, io: @io, seams: with_slow_turns, slot: slot)
|
|
100
102
|
slot_run.run(config_for(slot)).tap { @seams = slot_run.seams }
|
|
101
103
|
end
|
|
102
104
|
|
|
105
|
+
def with_slow_turns
|
|
106
|
+
return @seams if @seams.slow_turns
|
|
107
|
+
|
|
108
|
+
@seams.with(slow_turns: SlowTurns.new(Worktrees.new(@project).slow_turns_root))
|
|
109
|
+
end
|
|
110
|
+
|
|
103
111
|
def cancel_stale_pipelines
|
|
104
112
|
return unless recorder.db
|
|
105
113
|
|
|
@@ -34,9 +34,12 @@ module FunCi
|
|
|
34
34
|
# `process_table` answers what ps lists, for a stage over budget; `trunk`
|
|
35
35
|
# checks a commit against the trunk (#check(sha) answers a Trunk::Check,
|
|
36
36
|
# or nil for no check), nil for the project's own; `priorities` what jobs
|
|
37
|
-
# and the slow suite start under
|
|
37
|
+
# and the slow suite start under; `slow_turns` what a slow suite waits on
|
|
38
|
+
# while an older one of its branch runs (#take(branch) yields once it is
|
|
39
|
+
# its turn), nil for the project's own SlowTurns.
|
|
38
40
|
Seams = Data.define(:command_runner, :time_budgets, :commit_resolver, :recorder, :background_launcher, :workspace,
|
|
39
|
-
:stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk, :priorities
|
|
41
|
+
:stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk, :priorities,
|
|
42
|
+
:slow_turns)
|
|
40
43
|
|
|
41
44
|
# Reopened rather than given as a block to Data.define, so tools that read
|
|
42
45
|
# the source (mutineer) see these as Seams' methods.
|
|
@@ -52,7 +55,8 @@ module FunCi
|
|
|
52
55
|
workspace: nil, commit_resolver: method(:full_sha), stage_dir: StageDir.method(:create),
|
|
53
56
|
environment: ENV.to_h, clock: MONOTONIC,
|
|
54
57
|
extractor_runner: Evidence::CommandRunner.method(:new),
|
|
55
|
-
process_table: Evidence::ProcessTable.method(:now), trunk: nil, priorities: Priorities.for(RUBY_PLATFORM)
|
|
58
|
+
process_table: Evidence::ProcessTable.method(:now), trunk: nil, priorities: Priorities.for(RUBY_PLATFORM),
|
|
59
|
+
slow_turns: nil }
|
|
56
60
|
end
|
|
57
61
|
|
|
58
62
|
# The full SHA of the commit `rev` names, or nil when the repository has none.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require_relative "slot_lock"
|
|
5
|
+
|
|
6
|
+
module FunCi
|
|
7
|
+
module Pipeline
|
|
8
|
+
# A slow suite's place among its branch's: where its run's commit arrived
|
|
9
|
+
# (Persistence::Arrival), then its stage's id, so of two runs of one
|
|
10
|
+
# commit the later is the newer.
|
|
11
|
+
TurnTicket = Data.define(:place, :id) do
|
|
12
|
+
def name = format("%<place>020d-%<id>020d.lock", place: place, id: id)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# The tickets of a branch's slow suites waiting or running, as files in
|
|
16
|
+
# `dir`, each holding an flock on its own until its turn is over. A ticket
|
|
17
|
+
# appears already held (locked under a name of its own, then renamed), so
|
|
18
|
+
# one nobody holds was left by a suite that died, and goes.
|
|
19
|
+
class TurnQueue
|
|
20
|
+
# Blocks until the suite holding the ticket at each of `paths` is done.
|
|
21
|
+
WAIT = ->(paths) { paths.each { |path| TurnQueue.wait_on(path) } }
|
|
22
|
+
|
|
23
|
+
def self.wait_on(path)
|
|
24
|
+
File.open(path) { |file| file.flock(File::LOCK_SH) }
|
|
25
|
+
rescue Errno::ENOENT
|
|
26
|
+
nil
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def initialize(dir)
|
|
30
|
+
@dir = dir
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Holds `ticket` in the queue until the block returns.
|
|
34
|
+
def holding(ticket)
|
|
35
|
+
path = File.join(@dir, ticket.name)
|
|
36
|
+
file = enter(path)
|
|
37
|
+
yield
|
|
38
|
+
ensure
|
|
39
|
+
FileUtils.rm_f(path) if file
|
|
40
|
+
file&.close
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The tickets held of suites newer than `ticket`'s, as a Set of paths.
|
|
44
|
+
def newer_than(ticket)
|
|
45
|
+
Dir.glob(File.join(@dir, "*.lock")).select { |path| live?(path) && File.basename(path) > ticket.name }.to_set
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def enter(path)
|
|
51
|
+
FileUtils.mkdir_p(@dir)
|
|
52
|
+
locked("#{path}.#{Process.pid}.new").tap { |file| File.rename(file.path, path) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def locked(path) = File.new(path, File::RDWR | File::CREAT, 0o644).tap { |file| file.flock(File::LOCK_EX) }
|
|
56
|
+
|
|
57
|
+
def live?(path)
|
|
58
|
+
return true if SlotLock.held?(path)
|
|
59
|
+
|
|
60
|
+
FileUtils.rm_f(path)
|
|
61
|
+
false
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require_relative "slot_lock"
|
|
5
|
+
|
|
6
|
+
module FunCi
|
|
7
|
+
module Pipeline
|
|
8
|
+
# The slots held only by a slow suite waiting its branch's turn
|
|
9
|
+
# (acceptance-tests.md, AT-9.19). Such a suite keeps its slot, whose
|
|
10
|
+
# checkout and build it runs in, and holds an flock on the slot's mark
|
|
11
|
+
# under `waiting/` beside the slots while it waits. WorktreePool hands out
|
|
12
|
+
# a further slot for each mark held, so a queued suite keeps no other
|
|
13
|
+
# branch's run waiting. A mark dies with the process that held it.
|
|
14
|
+
module WaitingSlots
|
|
15
|
+
DIR = "waiting"
|
|
16
|
+
|
|
17
|
+
# The mark of the slot at `slot_path`, held until it is closed.
|
|
18
|
+
def self.mark(slot_path)
|
|
19
|
+
path = mark_path(slot_path)
|
|
20
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
21
|
+
File.new(path, File::RDWR | File::CREAT, 0o644).tap { |mark| mark.flock(File::LOCK_EX) }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# How many slots under `root` a waiting slow suite holds now.
|
|
25
|
+
def self.count(root) = Dir.glob(File.join(root, DIR, "*.lock")).count { |path| SlotLock.held?(path) }
|
|
26
|
+
|
|
27
|
+
def self.mark_path(slot_path) = File.join(File.dirname(slot_path), DIR, "#{File.basename(slot_path)}.lock")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -15,7 +15,8 @@ module FunCi
|
|
|
15
15
|
def self.for(project, sha)
|
|
16
16
|
return InPlace.new(project) if sha == NULL_SHA
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
config = Setup::ProjectConfig.new(project)
|
|
19
|
+
WorktreePool.new(Worktrees.new(project), size: config.worktree_slots, clean: config.clean)
|
|
19
20
|
end
|
|
20
21
|
end
|
|
21
22
|
end
|
|
@@ -3,27 +3,39 @@
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
require_relative "slot"
|
|
5
5
|
require_relative "worktrees"
|
|
6
|
+
require_relative "waiting_slots"
|
|
6
7
|
|
|
7
8
|
module FunCi
|
|
8
9
|
module Pipeline
|
|
9
10
|
# Hands each pipeline a worktree slot of its own under
|
|
10
|
-
#
|
|
11
|
-
# while every slot is taken.
|
|
11
|
+
# Worktrees#root, checked out at its commit, waiting
|
|
12
|
+
# while every slot is taken. A slot held only by a slow suite waiting its
|
|
13
|
+
# turn takes no place among `size` (WaitingSlots).
|
|
12
14
|
class WorktreePool
|
|
13
15
|
DEFAULT_SIZE = 2
|
|
16
|
+
# A slot nobody took for longer is cold: its caches are as warm as a
|
|
17
|
+
# run long ago left them, as a slot taken only when runs overlap is.
|
|
18
|
+
IDLE_SECONDS = 86_400
|
|
14
19
|
|
|
15
20
|
attr_reader :size
|
|
16
21
|
|
|
17
|
-
|
|
22
|
+
# Whether a slot last taken at `last_taken` has gone untaken too long.
|
|
23
|
+
def self.idle?(last_taken, now:) = now - last_taken > IDLE_SECONDS
|
|
24
|
+
|
|
25
|
+
# clean: the paths whose ignored files a slot loses when it moves off its
|
|
26
|
+
# line of history (Worktrees#check_out).
|
|
27
|
+
def initialize(worktrees, size: DEFAULT_SIZE, clean: [], waiter: -> { sleep 0.25 })
|
|
18
28
|
@worktrees = worktrees
|
|
19
29
|
@size = size
|
|
30
|
+
@clean = clean
|
|
20
31
|
@waiter = waiter
|
|
21
32
|
end
|
|
22
33
|
|
|
23
|
-
# A ColdSlot when the slot's worktree had to be made
|
|
34
|
+
# A ColdSlot when the slot's worktree had to be made, or was cleaned, or
|
|
35
|
+
# nobody took the slot for a day.
|
|
24
36
|
def acquire(sha)
|
|
25
37
|
slot = wait_for_free_slot
|
|
26
|
-
@worktrees.check_out(slot.path, sha) ? slot.as_cold : slot
|
|
38
|
+
@worktrees.check_out(slot.path, sha, clean: @clean) ? slot.as_cold : slot
|
|
27
39
|
rescue StandardError
|
|
28
40
|
slot&.release
|
|
29
41
|
raise
|
|
@@ -43,18 +55,26 @@ module FunCi
|
|
|
43
55
|
def free_slot
|
|
44
56
|
root = @worktrees.root
|
|
45
57
|
FileUtils.mkdir_p(root)
|
|
46
|
-
@size
|
|
58
|
+
working = @size + WaitingSlots.count(root)
|
|
59
|
+
working.times.lazy.filter_map { |index| try_lock(File.join(root, "slot-#{index}")) }.first
|
|
47
60
|
end
|
|
48
61
|
|
|
49
62
|
# The lock stays open for as long as the slot is held, so no block.
|
|
50
63
|
def try_lock(path)
|
|
51
64
|
lock = File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
|
|
52
|
-
return
|
|
65
|
+
return taken(path, lock) if lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
53
66
|
|
|
54
67
|
lock.close
|
|
55
68
|
nil
|
|
56
69
|
end
|
|
57
70
|
|
|
71
|
+
# Each taker stamps the lock, so its last change is when the slot was last taken.
|
|
72
|
+
def taken(path, lock)
|
|
73
|
+
idle = WorktreePool.idle?(lock.mtime, now: Time.now)
|
|
74
|
+
slot = Slot.new(path, stamped(lock), lock.path)
|
|
75
|
+
idle ? slot.as_cold : slot
|
|
76
|
+
end
|
|
77
|
+
|
|
58
78
|
def stamped(lock)
|
|
59
79
|
lock.truncate(0)
|
|
60
80
|
lock.write(Process.pid.to_s)
|
|
@@ -6,8 +6,8 @@ require_relative "worktrees"
|
|
|
6
6
|
module FunCi
|
|
7
7
|
module Pipeline
|
|
8
8
|
# Removes every slot of a project's worktree pool and every daily or
|
|
9
|
-
# weekly job's worktree,
|
|
10
|
-
# holds one. It takes each slot's lock first, so a run
|
|
9
|
+
# weekly job's worktree, those an older fun-ci kept under the git
|
|
10
|
+
# directory too, and git's entries for them, when no run or job holds one. It takes each slot's lock first, so a run
|
|
11
11
|
# starting meanwhile waits until the slot is gone and then checks it out
|
|
12
12
|
# afresh.
|
|
13
13
|
class WorktreePrune
|
|
@@ -43,8 +43,13 @@ module FunCi
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def slot_paths
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
patterns.flat_map { |pattern| Dir.glob(pattern) }.map { |path| path.delete_suffix(".lock") }.uniq
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def patterns
|
|
50
|
+
legacy = @worktrees.legacy_dir
|
|
51
|
+
[File.join(@worktrees.root, "slot-*"), File.join(@worktrees.jobs_root, "*"),
|
|
52
|
+
File.join(legacy, "worktrees", "slot-*"), File.join(legacy, "jobs", "*")]
|
|
48
53
|
end
|
|
49
54
|
end
|
|
50
55
|
end
|
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "open3"
|
|
4
|
+
require "digest"
|
|
4
5
|
require_relative "git_environment"
|
|
6
|
+
require_relative "../persistence/state_dir"
|
|
5
7
|
|
|
6
8
|
module FunCi
|
|
7
9
|
module Pipeline
|
|
8
|
-
# The git commands a worktree slot needs, run against the project's
|
|
10
|
+
# The git commands a worktree slot needs, run against the project's
|
|
11
|
+
# repository. The worktrees live in fun-ci's state directory, one place
|
|
12
|
+
# per repository (all its git worktrees share it), and not under .git/,
|
|
13
|
+
# which tools such as Vite refuse to serve files from.
|
|
9
14
|
class Worktrees
|
|
10
15
|
class GitError < StandardError; end
|
|
11
16
|
|
|
12
|
-
|
|
17
|
+
# state_dir: fun-ci's own (Persistence::StateDir).
|
|
18
|
+
def initialize(project_root, state_dir: Persistence::StateDir.path(ENV))
|
|
13
19
|
@project_root = project_root
|
|
20
|
+
@state_dir = state_dir
|
|
14
21
|
end
|
|
15
22
|
|
|
16
23
|
def root = File.join(fun_ci_dir, "worktrees")
|
|
@@ -18,15 +25,28 @@ module FunCi
|
|
|
18
25
|
# Where each daily and weekly job has a worktree of its own (Jobs::Locks).
|
|
19
26
|
def jobs_root = File.join(fun_ci_dir, "jobs")
|
|
20
27
|
|
|
28
|
+
# Where each branch's slow suites take turns (SlowTurns).
|
|
29
|
+
def slow_turns_root = File.join(fun_ci_dir, "slow")
|
|
30
|
+
|
|
31
|
+
# Where fun-ci before 2.4 kept the worktrees: under the git directory.
|
|
32
|
+
def legacy_dir = File.join(common_dir, "fun-ci")
|
|
33
|
+
|
|
21
34
|
# Leaves +path+ as a clean detached checkout of +sha+. Ignored files stay,
|
|
22
|
-
# so caches such as vendor/bundle survive between runs in the same slot
|
|
23
|
-
#
|
|
24
|
-
|
|
35
|
+
# so caches such as vendor/bundle survive between runs in the same slot,
|
|
36
|
+
# except under the +clean+ paths when +sha+ doesn't descend from the
|
|
37
|
+
# slot's last commit: those are outputs only as right as the commit that
|
|
38
|
+
# made them. Answers whether the slot is cold: a worktree it made, which
|
|
39
|
+
# has no caches, or one it cleaned.
|
|
40
|
+
def check_out(path, sha, clean: [])
|
|
25
41
|
return add(path, sha).then { true } unless File.exist?(File.join(path, ".git"))
|
|
26
42
|
|
|
43
|
+
moved_off = clean.any? && !descends?(path, sha)
|
|
27
44
|
git(path, "checkout", "--detach", "--force", sha)
|
|
28
45
|
git(path, "clean", "-fd")
|
|
29
|
-
false
|
|
46
|
+
return false unless moved_off
|
|
47
|
+
|
|
48
|
+
git(path, "clean", "-fdX", "--", *clean)
|
|
49
|
+
true
|
|
30
50
|
end
|
|
31
51
|
|
|
32
52
|
# Has git forget the worktrees whose directories are gone.
|
|
@@ -34,8 +54,22 @@ module FunCi
|
|
|
34
54
|
|
|
35
55
|
private
|
|
36
56
|
|
|
57
|
+
# <state>/projects/<repository's name>-<digest of its git directory>.
|
|
37
58
|
def fun_ci_dir
|
|
38
|
-
|
|
59
|
+
common = common_dir
|
|
60
|
+
name = File.basename(File.dirname(common))
|
|
61
|
+
File.join(@state_dir, "projects", "#{name}-#{Digest::SHA256.hexdigest(common)[0, 12]}")
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def common_dir
|
|
65
|
+
File.realpath(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root))
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Whether +sha+ descends from the commit checked out at +path+.
|
|
69
|
+
def descends?(path, sha)
|
|
70
|
+
last = git(path, "rev-parse", "HEAD").strip
|
|
71
|
+
_, status = Open3.capture2e(GitEnvironment::CLEAN, "git", "merge-base", "--is-ancestor", last, sha, chdir: path)
|
|
72
|
+
status.success?
|
|
39
73
|
end
|
|
40
74
|
|
|
41
75
|
def add(path, sha) = git(@project_root, "worktree", "add", "--detach", "--force", path, sha)
|
|
@@ -5,6 +5,7 @@ require_relative "hook_writer"
|
|
|
5
5
|
require_relative "git_hooks"
|
|
6
6
|
require_relative "setup_checker"
|
|
7
7
|
require_relative "legacy_hooks"
|
|
8
|
+
require_relative "init_options"
|
|
8
9
|
|
|
9
10
|
module FunCi
|
|
10
11
|
module Setup
|
|
@@ -18,13 +19,14 @@ module FunCi
|
|
|
18
19
|
end
|
|
19
20
|
|
|
20
21
|
def init(args)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
22
|
+
options = InitOptions.parse(args)
|
|
23
|
+
code = Installer.run(project_root: @project_root, stdout: @stdout, stack: options.stack)
|
|
24
|
+
return code if !code.zero? || !options.everything
|
|
25
|
+
|
|
26
|
+
hooks_then_check
|
|
27
|
+
rescue InitOptions::Invalid => e
|
|
28
|
+
@stdout.puts "fun-ci init: #{e.message}"
|
|
29
|
+
64
|
|
28
30
|
end
|
|
29
31
|
|
|
30
32
|
def install_hooks(args)
|
|
@@ -38,6 +40,13 @@ module FunCi
|
|
|
38
40
|
|
|
39
41
|
private
|
|
40
42
|
|
|
43
|
+
def hooks_then_check
|
|
44
|
+
code = install_hooks([])
|
|
45
|
+
return code unless code.zero?
|
|
46
|
+
|
|
47
|
+
check([])
|
|
48
|
+
end
|
|
49
|
+
|
|
41
50
|
def install(types)
|
|
42
51
|
hooks_dir = @hooks_dir.call(@project_root)
|
|
43
52
|
types.each do |type|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "stage_templates"
|
|
5
|
+
|
|
6
|
+
module FunCi
|
|
7
|
+
module Setup
|
|
8
|
+
# The build init writes for a JavaScript project that configures ESLint:
|
|
9
|
+
# the template's install, then ESLint, which loads what the install put
|
|
10
|
+
# in place, so it can't run in lint beside it (StageTemplates). nil keeps
|
|
11
|
+
# the template's build.
|
|
12
|
+
module EslintDetection
|
|
13
|
+
RUNNERS = { node_npm: "npx eslint .", node_pnpm: "pnpm exec eslint .", node_yarn: "yarn eslint .",
|
|
14
|
+
bun: "bunx eslint ." }.freeze
|
|
15
|
+
CONFIGS = %w[eslint.config.js eslint.config.mjs eslint.config.cjs eslint.config.ts eslint.config.mts
|
|
16
|
+
eslint.config.cts .eslintrc .eslintrc.js .eslintrc.cjs .eslintrc.json .eslintrc.yml
|
|
17
|
+
.eslintrc.yaml].freeze
|
|
18
|
+
|
|
19
|
+
# names: the files at the project's top; package_json: its text.
|
|
20
|
+
def self.build(stack, names, package_json)
|
|
21
|
+
runner = RUNNERS[stack]
|
|
22
|
+
return nil unless runner && configured?(names, package_json)
|
|
23
|
+
|
|
24
|
+
"#{StageTemplates.scripts(stack)["build.sh"].lines.last.chomp} && #{runner}"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def self.configured?(names, package_json) = names.intersect?(CONFIGS) || in_package?(package_json)
|
|
28
|
+
|
|
29
|
+
def self.in_package?(text)
|
|
30
|
+
parsed = JSON.parse(text)
|
|
31
|
+
parsed.is_a?(Hash) && parsed.key?("eslintConfig")
|
|
32
|
+
rescue JSON::ParserError
|
|
33
|
+
false
|
|
34
|
+
end
|
|
35
|
+
private_class_method :configured?, :in_package?
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Setup
|
|
7
|
+
# The arguments of `fun-ci init [--everything] [--stack NAME]`.
|
|
8
|
+
module InitOptions
|
|
9
|
+
class Invalid < StandardError; end
|
|
10
|
+
|
|
11
|
+
Parsed = Data.define(:everything, :stack)
|
|
12
|
+
|
|
13
|
+
def self.parse(args)
|
|
14
|
+
values = { everything: false, stack: nil }
|
|
15
|
+
left = parser(values).parse(args)
|
|
16
|
+
raise Invalid, "takes no #{left.join(" ")}" if left.any?
|
|
17
|
+
|
|
18
|
+
Parsed.new(**values)
|
|
19
|
+
rescue OptionParser::ParseError => e
|
|
20
|
+
raise Invalid, e.message
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Long options only, without OptionParser's own --help and --version.
|
|
24
|
+
def self.parser(values)
|
|
25
|
+
parser = OptionParser.new
|
|
26
|
+
parser.base.long.clear
|
|
27
|
+
parser.on("--everything") { values[:everything] = true }
|
|
28
|
+
parser.on("--stack NAME") { |name| values[:stack] = name }
|
|
29
|
+
parser
|
|
30
|
+
end
|
|
31
|
+
private_class_method :parser
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require_relative "project_detector"
|
|
4
4
|
require_relative "template_writer"
|
|
5
5
|
require_relative "lint_detection"
|
|
6
|
+
require_relative "eslint_detection"
|
|
7
|
+
require_relative "stage_templates"
|
|
6
8
|
require_relative "agent_instructions"
|
|
7
9
|
|
|
8
10
|
module FunCi
|
|
@@ -14,13 +16,18 @@ module FunCi
|
|
|
14
16
|
"Then fast.sh runs beside slow.sh, both on what build.sh built: have build.sh " \
|
|
15
17
|
"compile everything they need, test code included."].freeze
|
|
16
18
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
PLACEHOLDERS = "Could not detect the project type, so .fun-ci/ has placeholder scripts that pass and " \
|
|
20
|
+
"test nothing: put the project's own commands in them, or start again with --stack NAME."
|
|
21
|
+
|
|
22
|
+
# stack: a name of ProjectDetector::STACKS to write in place of the one detected.
|
|
23
|
+
def self.run(project_root:, stdout: $stdout, stack: nil)
|
|
24
|
+
new(project_root: project_root, stdout: stdout, stack: stack).run
|
|
19
25
|
end
|
|
20
26
|
|
|
21
|
-
def initialize(project_root:, stdout:)
|
|
27
|
+
def initialize(project_root:, stdout:, stack:)
|
|
22
28
|
@project_root = project_root
|
|
23
29
|
@stdout = stdout
|
|
30
|
+
@stack = stack
|
|
24
31
|
end
|
|
25
32
|
|
|
26
33
|
# A project set up for fun-ci, new or not, also tells agents what to do.
|
|
@@ -35,10 +42,26 @@ module FunCi
|
|
|
35
42
|
def set_up
|
|
36
43
|
return report(".fun-ci/ already exists, so init left it as it is.", 0) if initialised?
|
|
37
44
|
|
|
38
|
-
|
|
39
|
-
return
|
|
45
|
+
stack = chosen_stack
|
|
46
|
+
return unknown_stack if stack.nil?
|
|
47
|
+
return write_placeholders if stack == :unknown
|
|
48
|
+
|
|
49
|
+
write_templates(stack)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def chosen_stack
|
|
53
|
+
return ProjectDetector.new(Dir.children(@project_root)).detect if @stack.nil?
|
|
54
|
+
|
|
55
|
+
ProjectDetector::STACKS.keys.find { |stack| stack.to_s == @stack }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def unknown_stack
|
|
59
|
+
report("fun-ci init: no stack named #{@stack}; the stacks are #{ProjectDetector::STACKS.keys.join(", ")}", 64)
|
|
60
|
+
end
|
|
40
61
|
|
|
41
|
-
|
|
62
|
+
def write_placeholders
|
|
63
|
+
TemplateWriter.new(StageTemplates::PLACEHOLDERS, @project_root).write
|
|
64
|
+
report(PLACEHOLDERS, 0)
|
|
42
65
|
end
|
|
43
66
|
|
|
44
67
|
def tell_agents
|
|
@@ -58,11 +81,22 @@ module FunCi
|
|
|
58
81
|
|
|
59
82
|
def write_templates(detected)
|
|
60
83
|
@stdout.puts "Detected: #{detected.to_s.tr("_", " ")}"
|
|
61
|
-
TemplateWriter.new(
|
|
84
|
+
TemplateWriter.new(scripts_for(detected), @project_root).write
|
|
62
85
|
@stdout.puts "Created .fun-ci/ with template scripts.", *SIDE_BY_SIDE
|
|
63
86
|
0
|
|
64
87
|
end
|
|
65
88
|
|
|
89
|
+
def scripts_for(detected)
|
|
90
|
+
StageTemplates.scripts(detected, lint_override: LintDetection.command(detected, @project_root),
|
|
91
|
+
build_override: EslintDetection.build(detected, Dir.children(@project_root),
|
|
92
|
+
package_json))
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def package_json
|
|
96
|
+
path = File.join(@project_root, "package.json")
|
|
97
|
+
File.exist?(path) ? File.read(path) : "{}"
|
|
98
|
+
end
|
|
99
|
+
|
|
66
100
|
def report(message, exit_code)
|
|
67
101
|
@stdout.puts message
|
|
68
102
|
exit_code
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "stage_templates"
|
|
4
|
+
|
|
5
|
+
module FunCi
|
|
6
|
+
module Setup
|
|
7
|
+
# The stage scripts in .fun-ci/ that are still init's placeholders, as
|
|
8
|
+
# `fun-ci check`'s warnings.
|
|
9
|
+
class Placeholders
|
|
10
|
+
STAGES = %w[lint build fast slow].freeze
|
|
11
|
+
|
|
12
|
+
def initialize(project_root)
|
|
13
|
+
@dir = File.join(project_root, ".fun-ci")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def warnings = STAGES.select { |stage| placeholder?(stage) }.map { |stage| warning(stage) }
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def placeholder?(stage)
|
|
21
|
+
path = File.join(@dir, "#{stage}.sh")
|
|
22
|
+
File.file?(path) && File.read(path).include?(StageTemplates::MARKER)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def warning(stage)
|
|
26
|
+
".fun-ci/#{stage}.sh is init's placeholder, so the #{stage} stage tests nothing: put the project's own in it"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -49,6 +49,7 @@ module FunCi
|
|
|
49
49
|
def job_errors = Jobs::Folders.new(@project_root).errors
|
|
50
50
|
|
|
51
51
|
def worktree_slots = settings.worktree_slots
|
|
52
|
+
def clean = settings.clean
|
|
52
53
|
def trunk = settings.trunk
|
|
53
54
|
def trunk_fetch = settings.trunk_fetch
|
|
54
55
|
def job_spacing = settings.job_spacing
|
|
@@ -43,6 +43,11 @@ module FunCi
|
|
|
43
43
|
# and the jobs share the day evenly (Jobs::DueJobs).
|
|
44
44
|
def job_spacing = seconds(setting("job_spacing"))
|
|
45
45
|
|
|
46
|
+
# The paths, inside the project, whose git-ignored files a slot loses
|
|
47
|
+
# when it moves off its line of history (Pipeline::Worktrees); none
|
|
48
|
+
# when none are given, or what is given is no such list.
|
|
49
|
+
def clean = cleaned || []
|
|
50
|
+
|
|
46
51
|
def errors
|
|
47
52
|
setting_errors
|
|
48
53
|
rescue Psych::SyntaxError => e
|
|
@@ -58,7 +63,7 @@ module FunCi
|
|
|
58
63
|
def setting_errors
|
|
59
64
|
return [".fun-ci/config must be a mapping such as `worktree_slots: 2`"] unless raw.is_a?(Hash)
|
|
60
65
|
|
|
61
|
-
[slots_error, spacing_error].compact
|
|
66
|
+
[slots_error, spacing_error, clean_error].compact
|
|
62
67
|
end
|
|
63
68
|
|
|
64
69
|
# The slots the file asks for, when that is a whole number above 0.
|
|
@@ -80,6 +85,21 @@ module FunCi
|
|
|
80
85
|
".fun-ci/config: job_spacing must be seconds, or a number with s, m or h, not #{given.inspect}"
|
|
81
86
|
end
|
|
82
87
|
|
|
88
|
+
def cleaned
|
|
89
|
+
given = setting("clean")
|
|
90
|
+
given if given.is_a?(Array) && given.all? { |path| inside?(path) }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def inside?(path) = path.is_a?(String) && !path.start_with?("/") && !path.split("/").include?("..")
|
|
94
|
+
|
|
95
|
+
def clean_error
|
|
96
|
+
given = setting("clean")
|
|
97
|
+
return nil if given.nil? || cleaned
|
|
98
|
+
|
|
99
|
+
".fun-ci/config: clean must be a list of paths inside the project, such as [public/assets], " \
|
|
100
|
+
"not #{given.inspect}"
|
|
101
|
+
end
|
|
102
|
+
|
|
83
103
|
def setting(key)
|
|
84
104
|
raw.is_a?(Hash) ? raw[key] : nil
|
|
85
105
|
rescue Psych::Exception
|
|
@@ -4,19 +4,21 @@ require_relative "project_config"
|
|
|
4
4
|
require_relative "legacy_hooks"
|
|
5
5
|
require_relative "installed_hooks"
|
|
6
6
|
require_relative "trunk_report"
|
|
7
|
+
require_relative "placeholders"
|
|
7
8
|
|
|
8
9
|
module FunCi
|
|
9
10
|
module Setup
|
|
10
11
|
class SetupChecker
|
|
11
|
-
#
|
|
12
|
-
#
|
|
12
|
+
# What it warns of: fun-ci 1.x's hooks, any in `hooks_dir` that doesn't
|
|
13
|
+
# run fun-ci, and stage scripts still init's placeholders.
|
|
13
14
|
Hooks = Data.define(:sources) do
|
|
14
15
|
def warnings = sources.flat_map(&:warnings)
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
# hooks_dir: where git runs the project's hooks from; nil outside a git repository.
|
|
18
19
|
def self.run(project_root:, stdout: $stdout, hooks_dir: nil)
|
|
19
|
-
|
|
20
|
+
sources = [LegacyHooks.new(project_root), InstalledHooks.new(hooks_dir), Placeholders.new(project_root)]
|
|
21
|
+
hooks = Hooks.new(sources)
|
|
20
22
|
new(config: ProjectConfig.new(project_root), hooks: hooks, stdout: stdout,
|
|
21
23
|
trunk: TrunkReport.new(project_root)).run
|
|
22
24
|
end
|