fun_ci 2.1.1-aarch64-linux → 2.2.0-aarch64-linux

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -5,24 +5,41 @@ require_relative "../persistence/pipeline_recorder"
 require_relative "trigger_params"
 require_relative "../setup/project_config"
 require_relative "../trunk/first_fetch"
+require_relative "../jobs/due_jobs"
+require_relative "../jobs/job_fork"
 
 module FunCi
   module Pipeline
     class PipelineForker
-      # A run started, and what to say before the project's first fetch of its trunk, if this is it.
-      Forked = Data.define(:notice)
+      # A run started, what to say before the project's first fetch of its
+      # trunk, if this is it, and why the daily and weekly jobs didn't start,
+      # if they didn't.
+      Forked = Data.define(:notice, :jobs)
+
+      class Forked
+        def initialize(notice:, jobs: nil) = super
+      end
 
       # Answers a Forked, or false in a project not set up for fun-ci. What the
-      # run prints goes nowhere, so the notice is worked out here, before the fork.
-      def self.fork_pipeline(commit_hash:, branch:, db_path:)
+      # run prints goes nowhere, so the notice is worked out here, before the
+      # fork. The project's due jobs start beside the run, each in a process
+      # of its own (design.md, Daily and weekly jobs), once the run has
+      # started, and trouble starting them never stops it. jobs: what starts
+      # them, given the commit and the database's path.
+      def self.fork_pipeline(commit_hash:, branch:, db_path:, jobs: method(:start_due_jobs))
         return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
 
         notice = first_fetch(db_path)
-        pid = fork do
-          run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
-        end
-        Process.detach(pid)
-        Forked.new(notice: notice)
+        Process.detach(fork { run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path) })
+        Forked.new(notice: notice, jobs: trouble { jobs.call(Commit.new(sha: commit_hash, branch: branch), db_path) })
+      end
+
+      # What went wrong starting the jobs, or nil.
+      def self.trouble
+        yield
+        nil
+      rescue StandardError => e
+        "fun-ci: the daily and weekly jobs didn't start: #{e.message}"
       end
 
       # The connection closes before the fork, which must not inherit it.
@@ -33,6 +50,18 @@ module FunCi
         db&.close
       end
 
+      # Which jobs are due is read before any fork, which must not inherit the connection.
+      def self.start_due_jobs(commit, db_path)
+        due_jobs(db_path).each { |job| Jobs::JobFork.start(job, commit, project: Dir.pwd, db_path: db_path) }
+      end
+
+      def self.due_jobs(db_path)
+        db = Persistence::Database.connection(db_path)
+        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).list
+      ensure
+        db&.close
+      end
+
       # Trigger requires this file, through TriggerCommand, so it is loaded
       # here, where it is used, rather than above.
       def self.run_in_child(commit_hash:, branch:, db_path:)
@@ -49,7 +78,7 @@ module FunCi
         Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
                     io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
-      private_class_method :trigger, :first_fetch
+      private_class_method :trigger, :first_fetch, :start_due_jobs, :due_jobs, :trouble
     end
   end
 end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -23,16 +23,24 @@ module FunCi
       DRAIN_SECONDS = 1
 
       # Where the command runs, what it finds in its environment besides
-      # fun-ci's own, the window what it prints is written to, and what to do
-      # with its pid when it runs over budget, before it is killed.
-      Launch = Data.define(:chdir, :env, :output, :before_kill)
+      # fun-ci's own, the window what it prints is written to, what to do
+      # with its pid when it runs over budget, before it is killed, and the
+      # open files it holds as long as it runs, such as a lock.
+      Launch = Data.define(:chdir, :env, :output, :before_kill, :held)
 
       class Launch
         def initialize(**given)
-          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, **given)
+          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], **given)
         end
       end
 
+      # Where the first file a command holds is open in it: past the gate's 9.
+      FIRST_HELD = 10
+
+      # { descriptor in the command => file } for the files it holds. Not their
+      # own descriptors, which in a process with few files open may be the gate's.
+      def self.held_descriptors(files) = files.each_with_index.to_h { |file, index| [FIRST_HELD + index, file] }
+
       # A command started and let go, and the pipe that carries what it prints.
       Started = Data.define(:pid, :reader)
 
@@ -70,9 +78,11 @@ module FunCi
 
       def spawn_gated(cmd, writer, gate, launch)
         Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
-                      out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir)
+                      out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir, **held(launch))
       end
 
+      def held(launch) = ProcessRunner.held_descriptors(launch.held)
+
       # The reader may be closed while this thread still reads, once a killed
       # command's output has drained or stopped coming.
       def read_until_closed(reader, printed)

data/lib/fun_ci/pipeline/run_canceller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "../persistence/active_runs"
+require_relative "../persistence/active_jobs"
 require_relative "../persistence/run_status"
 require_relative "../persistence/write_trouble"
 require_relative "slot_lock"
@@ -36,6 +37,26 @@ module FunCi
         # The next poll looks again.
       end
 
+      # Stops a daily or weekly job's processes and records it cancelled,
+      # unless it has finished meanwhile; a group its runner recorded while
+      # this looked is stopped once the runner is dead, as a run's late stages are.
+      def cancel_job(db, id)
+        Persistence::ActiveJobs.with_id(db, id).each do |job|
+          stop(job)
+          Persistence::ActiveJobs.with_id(db, id).each { |now| kill_late(job, now) }
+          Persistence::ActiveJobs.cancelled(db, id)
+        end
+      end
+
+      # Records failed each daily or weekly job still running whose lock
+      # nobody holds: its process is gone (acceptance-tests.md, AT-13.6).
+      def record_dead_jobs(db)
+        Persistence::ActiveJobs.running(db).filter_map { |id, lock| id unless @slot_held.call(lock) }
+                               .each { |id| Persistence::ActiveJobs.died(db, id) }
+      rescue SQLite3::BusyException, *Persistence::WriteTrouble::UNWRITABLE
+        # The next poll looks again.
+      end
+
       def stop(run)
         return unless alive?(run)
 
@@ -53,6 +74,9 @@ module FunCi
         end
       end
 
+      # The groups recorded since `before` was read.
+      def kill_late(before, now) = (now.stage_groups - before.stage_groups).each { |group| kill(-group) }
+
       def alive?(run) = run.slot_lock.nil? || @slot_held.call(run.slot_lock)
 
       # Whether a process with this pid exists; one this user may not signal does.

data/lib/fun_ci/pipeline/slot.rb

@@ -6,7 +6,7 @@ module FunCi
     # file. The fast suite and the slow suite each hold it; it frees when the
     # last of them lets go. A process that dies lets go with it.
     class Slot
-      attr_reader :path, :lock_file
+      attr_reader :path, :lock_file, :lock
 
       # +lock_file+ names the lock on disk; nil for a slot with nothing to lock.
       def initialize(path, lock, lock_file = nil)

data/lib/fun_ci/pipeline/stage_execution.rb

@@ -7,12 +7,18 @@ module FunCi
   module Pipeline
     # Runs a stage that has been started, in `dir`, with a stage directory of
     # its own, and records how it ended while it still holds both. The same
-    # for every stage, the slow suite in its forked child included.
+    # for every stage, the slow suite in its forked child included, and for
+    # a daily or weekly job.
     class StageExecution
-      def initialize(seams:, dir:, commit:)
+      # launching: what the script is launched with besides its output window
+      # (ProcessRunner::Launch): what it is told about itself, by default its
+      # stage's name in FUN_CI_STAGE (acceptance-tests.md, AT-10.7), and the
+      # files it holds open.
+      def initialize(seams:, dir:, commit:, launching: {})
         @seams = seams
         @dir = dir
         @commit = commit
+        @launching = launching
       end
 
       # Answers [the outcome recorded, what the stage printed].
@@ -36,10 +42,9 @@ module FunCi
         StageEnd::Finished.new(output: output, status: status, timed_out: timed_out, overrun: overrun)
       end
 
-      # The stage learns its name from FUN_CI_STAGE (acceptance-tests.md, AT-10.7).
       def execute(stage, command, stage_dir, before_kill, &)
         window = stage_dir.window
-        launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, output: window,
+        launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, **@launching, output: window,
                                            before_kill: before_kill)
         @seams.executor(@dir).call(command, @seams.budgets[stage], launch, &)
       ensure

data/lib/fun_ci/pipeline/trigger_command.rb

@@ -56,10 +56,16 @@ module FunCi
         return 0 unless forked
 
         say_how_to_wait(commit.sha[0, 7])
-        @io.stdout.puts(forked.notice) if forked.notice
+        say(forked)
         0
       end
 
+      # The first fetch's notice, and on stderr, why the jobs didn't start.
+      def say(forked)
+        @io.stdout.puts(forked.notice) if forked.notice
+        @io.stderr.puts(forked.jobs) if forked.jobs
+      end
+
       def say_how_to_wait(sha) = @io.stdout.puts("fun-ci: testing #{sha}. Verdict: fun-ci wait #{sha} --need all")
     end
   end

data/lib/fun_ci/pipeline/worktree_prune.rb

@@ -5,8 +5,9 @@ require_relative "worktrees"
 
 module FunCi
   module Pipeline
-    # Removes every slot of a project's worktree pool, and git's entries for
-    # them, when no run holds one. It takes each slot's lock first, so a run
+    # Removes every slot of a project's worktree pool and every daily or
+    # weekly job's worktree, and git's entries for them, when no run or job
+    # holds one. It takes each slot's lock first, so a run
     # starting meanwhile waits until the slot is gone and then checks it out
     # afresh.
     class WorktreePrune
@@ -16,8 +17,8 @@ module FunCi
         @worktrees = worktrees
       end
 
-      # How many slots it removed. Raises Busy, removing nothing, while a run
-      # holds a slot.
+      # How many worktrees it removed. Raises Busy, removing nothing, while a
+      # run or a job holds one.
       def run
         locks = take_locks
         locks.each { |lock| FileUtils.rm_rf(lock.path.delete_suffix(".lock")) }
@@ -37,12 +38,13 @@ module FunCi
           next if locks.last.flock(File::LOCK_EX | File::LOCK_NB)
 
           locks.each(&:close)
-          raise Busy, "a pipeline is running in #{path}"
+          raise Busy, "fun-ci is running something in #{path}"
         end
       end
 
       def slot_paths
-        Dir.glob(File.join(@worktrees.root, "slot-*")).map { |path| path.delete_suffix(".lock") }.uniq
+        (Dir.glob(File.join(@worktrees.root, "slot-*")) + Dir.glob(File.join(@worktrees.jobs_root, "*")))
+          .map { |path| path.delete_suffix(".lock") }.uniq
       end
     end
   end

data/lib/fun_ci/pipeline/worktrees.rb

@@ -13,10 +13,10 @@ module FunCi
         @project_root = project_root
       end
 
-      def root
-        File.join(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root),
-                  "fun-ci", "worktrees")
-      end
+      def root = File.join(fun_ci_dir, "worktrees")
+
+      # Where each daily and weekly job has a worktree of its own (Jobs::Locks).
+      def jobs_root = File.join(fun_ci_dir, "jobs")
 
       # Leaves +path+ as a clean detached checkout of +sha+. Ignored files stay,
       # so caches such as vendor/bundle survive between runs in the same slot.
@@ -32,6 +32,10 @@ module FunCi
 
       private
 
+      def fun_ci_dir
+        File.join(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root), "fun-ci")
+      end
+
       def add(path, sha) = git(@project_root, "worktree", "add", "--detach", "--force", path, sha)
 
       def git(dir, *args)

data/lib/fun_ci/setup/agent_instructions.rb

@@ -19,6 +19,9 @@ module FunCi
         the commit conflicts with the trunk, which has to be integrated first,
         as it prints.
         `fun-ci runs` lists recent runs, and `fun-ci status` says where one stands.
+        `fun-ci jobs` lists the daily and weekly jobs, which run beside the
+        pipeline and never hold up a verdict, and `fun-ci why --job NAME` says
+        why one failed.
       MD
 
       def self.file_for(names) = names.include?("CLAUDE.md") && !names.include?("AGENTS.md") ? "CLAUDE.md" : "AGENTS.md"

data/lib/fun_ci/setup/project_config.rb

@@ -4,6 +4,7 @@ require "shellwords"
 require_relative "settings"
 require_relative "../evidence/settings"
 require_relative "../evidence/start"
+require_relative "../jobs/folders"
 
 module FunCi
   module Setup
@@ -37,6 +38,11 @@ module FunCi
       # may hold, such as JSON logs.
       def any_project_presets = preset_candidates.reject(&:because).map { |found| found.preset.name }
 
+      # The daily and weekly jobs that can run, and why any other can't
+      # (design.md, Daily and weekly jobs); neither stops a pipeline.
+      def jobs = Jobs::Folders.new(@project_root).jobs
+      def job_errors = Jobs::Folders.new(@project_root).errors
+
       def worktree_slots = settings.worktree_slots
       def trunk = settings.trunk
       def trunk_fetch = settings.trunk_fetch

data/lib/fun_ci/setup/setup_checker.rb

@@ -22,10 +22,11 @@ module FunCi
 
       # Warnings are printed but don't fail the check.
       def run
-        errors = @config.validate + @config.evidence_errors
+        errors = @config.validate + @config.evidence_errors + @config.job_errors
         @stdout.puts(errors.empty? ? "All OK. The project is configured." : errors)
         @stdout.puts(@trunk.lines)
         list_presets
+        list_jobs
         @hooks.warnings.each { |warning| @stdout.puts "Warning: #{warning}" }
         errors.empty? ? 0 : 1
       end
@@ -36,6 +37,11 @@ module FunCi
         lists = { "this project" => @config.presets, "any project's output" => @config.any_project_presets }
         lists.each { |what, names| @stdout.puts "Evidence presets for #{what}: #{names.join(", ")}" if names.any? }
       end
+
+      def list_jobs
+        jobs = @config.jobs
+        @stdout.puts "Jobs: #{jobs.map { |job| "#{job.name} (#{job.cadence})" }.join(", ")}" if jobs.any?
+      end
     end
   end
 end

data/lib/fun_ci/version.rb

@@ -2,5 +2,5 @@
 
 # Loaded on its own by the gemspec, so it must not require anything.
 module FunCi
-  VERSION = "2.1.1"
+  VERSION = "2.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fun_ci
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: aarch64-linux
 authors:
 - Erik Thyge Madsen
@@ -67,11 +67,20 @@ files:
 - lib/fun_ci/agent/commands.rb
 - lib/fun_ci/agent/context.rb
 - lib/fun_ci/agent/digest.rb
+- lib/fun_ci/agent/due_in.rb
 - lib/fun_ci/agent/events.rb
 - lib/fun_ci/agent/events_command.rb
 - lib/fun_ci/agent/evidence_text.rb
 - lib/fun_ci/agent/exit_code.rb
 - lib/fun_ci/agent/git.rb
+- lib/fun_ci/agent/job_events.rb
+- lib/fun_ci/agent/job_json.rb
+- lib/fun_ci/agent/job_report.rb
+- lib/fun_ci/agent/job_why.rb
+- lib/fun_ci/agent/job_why_json.rb
+- lib/fun_ci/agent/job_why_text.rb
+- lib/fun_ci/agent/jobs_command.rb
+- lib/fun_ci/agent/jobs_text.rb
 - lib/fun_ci/agent/live_pipeline.rb
 - lib/fun_ci/agent/next_step.rb
 - lib/fun_ci/agent/options.rb
@@ -83,6 +92,7 @@ files:
 - lib/fun_ci/agent/runs_command.rb
 - lib/fun_ci/agent/runs_text.rb
 - lib/fun_ci/agent/snapshots.rb
+- lib/fun_ci/agent/span.rb
 - lib/fun_ci/agent/stage_summary.rb
 - lib/fun_ci/agent/status_command.rb
 - lib/fun_ci/agent/status_json.rb
@@ -110,10 +120,14 @@ files:
 - lib/fun_ci/console/console_loop.rb
 - lib/fun_ci/console/console_session.rb
 - lib/fun_ci/console/console_state.rb
+- lib/fun_ci/console/job_message.rb
+- lib/fun_ci/console/job_order.rb
+- lib/fun_ci/console/job_rows.rb
 - lib/fun_ci/console/key_handler.rb
 - lib/fun_ci/console/launcher.rb
 - lib/fun_ci/console/milestone_events.rb
 - lib/fun_ci/console/milestones.rb
+- lib/fun_ci/console/paging.rb
 - lib/fun_ci/console/renderer_lookup.rb
 - lib/fun_ci/console/renderer_process.rb
 - lib/fun_ci/console/row_order.rb
@@ -205,8 +219,21 @@ files:
 - lib/fun_ci/evidence/stamp.rb
 - lib/fun_ci/evidence/start.rb
 - lib/fun_ci/evidence/worktree.rb
+- lib/fun_ci/jobs/due.rb
+- lib/fun_ci/jobs/due_jobs.rb
+- lib/fun_ci/jobs/folders.rb
+- lib/fun_ci/jobs/job.rb
+- lib/fun_ci/jobs/job_fork.rb
+- lib/fun_ci/jobs/job_run.rb
+- lib/fun_ci/jobs/locks.rb
+- lib/fun_ci/jobs/site.rb
+- lib/fun_ci/jobs/standings.rb
+- lib/fun_ci/jobs/state.rb
+- lib/fun_ci/persistence/active_jobs.rb
 - lib/fun_ci/persistence/active_runs.rb
 - lib/fun_ci/persistence/database.rb
+- lib/fun_ci/persistence/job_recorder.rb
+- lib/fun_ci/persistence/job_runs.rb
 - lib/fun_ci/persistence/null_recorder.rb
 - lib/fun_ci/persistence/output_tail.rb
 - lib/fun_ci/persistence/pipeline_recorder.rb