asgard 0.5.1 → 0.6.0

data/lib/asgard/jobs/stopper.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Asgard
+  module Jobs
+    # Ends one job: TERM to its process group, a poll every 0.2s for up to
+    # the grace period, then KILL and a wait until the group holds none of
+    # the job's processes. The signal sender, the liveness check and
+    # sleep are injectable so the timing branches test without waiting.
+    class Stopper
+      POLL        = 0.2
+      TIMEOUT     = 10
+      WARN_AFTER  = 1.0
+      RESEND_EVERY = 1.0
+
+      # A negative pid addresses a group, and kill(2) gives -1 and 0 their
+      # own meanings: every process the user owns, and the caller's own
+      # group. A record that is corrupt or hand-written must never turn
+      # into either, so only a real group id (above 1) is ever signalled.
+      #
+      # A valid id is still refused (false) when it is our own group, or
+      # when no process leads it any more: a job's pgid is its trampoline's
+      # pid, so a group whose leader is gone or moved is not the job. So is
+      # a group we may not signal (EPERM): its id now belongs to another
+      # user's processes. +kill+ is Process.kill, injectable for tests.
+      def self.signal_group(signal, pgid, own_pgrp: Process.getpgrp, pgid_of: Process.method(:getpgid),
+                            kill: Process.method(:kill))
+        unless pgid.is_a?(Integer) && pgid > 1
+          raise ArgumentError,
+                "refusing to signal process group #{pgid.inspect}: not a group id above 1"
+        end
+        return false unless led_or_orphaned?(pgid, own_pgrp:, pgid_of:, kill:)
+
+        kill.call(signal, -pgid)
+      rescue Errno::ESRCH
+        nil
+      rescue Errno::EPERM
+        false
+      end
+
+      # A group may be signalled when its leader (the trampoline) still
+      # leads it, or when the leader is gone but members remain: then the
+      # id cannot have been reused. Never our own group, and never one we
+      # may not signal (EPERM). ESRCH from the group check means nobody is
+      # left: nil from signal_group.
+      def self.led_or_orphaned?(pgid, own_pgrp:, pgid_of:, kill: Process.method(:kill))
+        return false if pgid == own_pgrp
+
+        leader_group = begin
+          pgid_of.call(pgid)
+        rescue Errno::ESRCH
+          kill.call(0, -pgid) # members without their leader, or ESRCH
+          pgid
+        end
+        safe_group?(pgid, pid: pgid, own_pgrp:, pgid_of: ->(_) { leader_group })
+      rescue Errno::EPERM
+        false
+      end
+
+      # Whether +pgid+ may be signalled as the group +pid+ leads: a group id
+      # above 1, not our own, and still the group +pid+ belongs to.
+      def self.safe_group?(pgid, pid:, own_pgrp:, pgid_of:)
+        return false unless pgid.is_a?(Integer) && pgid > 1 && pgid != own_pgrp
+
+        pgid_of.call(pid) == pgid
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      # +alive+ is State.active?: a job whose script exited still counts
+      # while processes it left run in its group, and ps has confirmed
+      # they are there before any signal goes out.
+      def initialize(timeout: TIMEOUT, poll: POLL, out: $stdout,
+                     alive: State.method(:active?), signal: Stopper.method(:signal_group), sleeper: Kernel.method(:sleep))
+        @timeout = timeout
+        @poll    = poll
+        @out     = out
+        @alive   = alive
+        @signal  = signal
+        @sleeper = sleeper
+      end
+
+      # :stopped when TERM ended the job within the grace period, :killed
+      # when KILL had to, :not_running when there was nothing to stop. A
+      # KILL takes the trampoline too, so the 137 is recorded here (a
+      # record that is not a job's, as the timing tests use, writes none).
+      def stop(record)
+        return :not_running unless @alive.call(record)
+
+        @signal.call("TERM", record[:pgid])
+        return :stopped if exited?(record, resend: "TERM")
+
+        @signal.call("KILL", record[:pgid])
+        State.record_kill(record) if exited?(record)
+        :killed
+      end
+
+      # Whether poll number +tick+ (from 0) re-sends TERM: once every
+      # RESEND_EVERY seconds. A TERM that reached the trampoline's child
+      # between its fork and its exec met the inherited trap and was lost;
+      # by the next one the child runs the job with the default handler.
+      def self.resend_tick?(tick, poll:)
+        every = (RESEND_EVERY / poll).round
+        tick.positive? && (tick % every).zero?
+      end
+
+      # Stops and reports one job; the outcome as above.
+      def stop_and_report(record)
+        outcome = stop(record)
+        @out.puts report_line(outcome, record)
+        outcome
+      end
+
+      def report_line(outcome, record)
+        name, pid = record.values_at(:name, :pid)
+        case outcome
+        when :stopped then "stopped #{name} (pid #{pid})"
+        when :killed  then "killed #{name} (pid #{pid}) after #{@timeout}s"
+        else               "#{name} is not running"
+        end
+      end
+
+      private
+
+      # True once the job is gone, false when the grace period ran out.
+      # Says what it is waiting for once, when the wait reaches a second
+      # (or its last poll, for a shorter grace period).
+      def exited?(record, resend: nil)
+        ticks = (@timeout / @poll).ceil
+        warn_tick = [(WARN_AFTER / @poll).round, ticks].min - 1
+        ticks.times do |tick|
+          return true unless @alive.call(record)
+
+          @signal.call(resend, record[:pgid]) if resend && Stopper.resend_tick?(tick, poll: @poll)
+          @sleeper.call(@poll)
+          @out.puts "waiting for #{record[:name]} (pid #{record[:pid]})" if tick == warn_tick
+        end
+        !@alive.call(record)
+      end
+    end
+  end
+end

data/lib/asgard/jobs.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "jobs/declaration"
+require_relative "jobs/state"
+require_relative "jobs/launcher"
+require_relative "jobs/stopper"
+require_relative "jobs/commands"
+
+module Asgard
+  # Background jobs started from a task body:
+  #
+  #   class Tasks
+  #     def doc_server = sh "mkdocs serve", background: true   # doc_server-1
+  #     def servers
+  #       sh "docker compose up postgres", background: true     # servers-1
+  #       sh "bin/rails server",            background: :api    # api
+  #     end
+  #   end
+  #
+  # then manage them with `asgard jobs list|stop|log|restart|clean`.
+  module Jobs
+    # A job could not be managed: an unknown name, no owning task.
+    class Error < Asgard::Error; end
+  end
+end

data/lib/asgard/kernel_methods.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require "pathname"
-
+# Every helper defined here is also in Asgard::Base::KERNEL_HELPERS, so a
+# task or helper by its name is refused rather than hiding it.
 module Kernel
   def debug?   = $DEBUG
   def verbose? = $VERBOSE
@@ -26,7 +26,7 @@ module Kernel
     dir = Pathname.new(Dir.pwd)
     loop do
       candidate = dir + name
-      return candidate if candidate.exist?
+      return candidate if candidate.file? # a directory by that name is not it
       parent = dir.parent
       break if parent == dir
       dir = parent
@@ -39,36 +39,41 @@ module Kernel
   # defaults to the immediate caller, but is exposed as a keyword so a
   # wrapper (e.g. Asgard::Doctor's import tracer, prepended via `super`)
   # can thread through the *real* caller instead of its own frame.
+  #
+  # Only +path+ as written decides whether it is a glob. The caller's
+  # directory is the glob's base, never part of the pattern, so a project
+  # directory named like a glob ("glob[1]") does not change what matches.
   def import(path, from: caller_locations(1, 1).first)
     path = path.to_s
     raise ArgumentError, "import: path must end with .loki (got #{path.inspect})" unless path.end_with?(".loki")
-    unless File.absolute_path?(path)
-      caller_dir = File.dirname(from.absolute_path)
-      path = File.expand_path(path, caller_dir)
-    end
-    paths = path =~ /[*?\[{]/ ? Dir.glob(path) : [path]
-    loaded = paths.map do |p|
+    base   = File.dirname(from.absolute_path) unless File.absolute_path?(path)
+    paths  = Asgard.glob?(path) ? Dir.glob(path, base:) : [path]
+    loaded = paths.map { File.expand_path(it, base) }.map do |p|
       if $LOADED_FEATURES.include?(p)
         warn "import: skip #{p} (already loaded)" if debug?
         next false
       end
       warn "import: #{p}" if verbose? || debug?
-      load p
+      # Recorded before it loads, as Asgard.load_root does, so two files
+      # that import each other each load once instead of without end.
       $LOADED_FEATURES << p
+      load p
       true
     end
     loaded.any?
   end
   module_function :import
 
+  # The paths found are handed to import escaped: the directory they sit
+  # in is part of a path, not a pattern.
   def import_up(name = ".loki")
-    if name =~ /[*?\[{]/
+    if Asgard.glob?(name)
       dir = Dir.pwd
       loop do
-        matches = Dir.glob(File.join(dir, name))
+        matches = Dir.glob(name, base: dir)
         unless matches.empty?
           warn "import_up: #{name} → #{dir}" if verbose? || debug?
-          return matches.map { |p| import(p) }.any?
+          return matches.map { |p| import(Asgard.glob_escape(File.join(dir, p))) }.any?
         end
         parent = File.dirname(dir)
         break if parent == dir
@@ -83,7 +88,7 @@ module Kernel
       return false
     end
     warn "import_up: #{name} → #{path}" if verbose? || debug?
-    import path
+    import Asgard.glob_escape(path)
   end
   module_function :import_up
 end

data/lib/asgard/leading_flag.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Loaded alone by bin/asgard to answer --version before Thor loads, so it
+# requires nothing.
+module Asgard
+  # Whether +flag+ is one of asgard's own options: among the options
+  # before the first word that is not one (the task name). After it, the
+  # word belongs to the task, which may take --version as an argument.
+  def self.leading_flag?(argv, flag) = argv.take_while { |word| word.start_with?("-") }.include?(flag)
+end

data/lib/asgard/pid.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Asgard
+  # Questions about a pid that outlived the process that recorded it: is
+  # it alive, what is it running, and is it still the process we started?
+  # The watcher and background jobs both keep pids on disk across asgard
+  # runs, and a pid left by a crash or a reboot can belong to an unrelated
+  # process by the time it is read back. (Not Asgard::Process: that would
+  # shadow ::Process inside the namespace.)
+  module Pid
+    # ps by absolute path: a scheduled or watched task's PATH may not
+    # hold it.
+    PS = %w[/bin/ps /usr/bin/ps].find { |path| File.executable?(path) } || "ps"
+
+    # The system's ps cannot answer what these questions ask of it.
+    class Unusable < Asgard::Error; end
+
+    # One process of a group: its pid, when it started (to ps's one-second
+    # resolution) and its command line.
+    Member = Data.define(:pid, :started, :command)
+
+    module_function
+
+    # Whether a process exists; one we may not signal still exists.
+    def alive?(pid)
+      Process.kill(0, pid)
+      true
+    rescue Errno::ESRCH
+      false
+    rescue Errno::EPERM
+      true
+    end
+
+    # The ps argv for one pid's full command line. -ww: procps cuts the
+    # line to the terminal width (COLUMNS) without it, and a job's script
+    # path sits at the very end of a long one. macOS accepts it too.
+    def ps_argv(pid) = [PS, "-ww", "-o", "command=", "-p", pid.to_s]
+
+    # The command line of a live process as `ps` prints it, or nil when
+    # there is no such process. ps is the one way that works unchanged
+    # on macOS and Linux.
+    def command_line(pid)
+      out, status = Open3.capture2(*ps_argv(pid))
+      status.success? ? out.strip : nil
+    rescue SystemCallError
+      nil
+    end
+
+    # The command lines of every process in group +pgid+. ps by group id
+    # is not portable, so this lists every process with its group and
+    # keeps the matching ones (see group_commands).
+    def group_command_lines(pgid)
+      out, status = Open3.capture2(PS, "-ww", "-A", "-o", "pgid=,command=")
+      status.success? ? group_commands(out, pgid) : []
+    rescue SystemCallError
+      []
+    end
+
+    # The commands in `ps -o pgid=,command=` output that belong to +pgid+.
+    def group_commands(ps_output, pgid)
+      ps_output.each_line.filter_map do |line|
+        group, command = line.strip.split(/\s+/, 2)
+        command.to_s if group.to_i == pgid
+      end
+    end
+
+    # Every process in group +pgid+ as Members. Like group_command_lines,
+    # but with each one's age, so a caller can tell a group that has kept
+    # its members from a later group that reused the id.
+    def group_processes(pgid, now: Time.now)
+      out, status = Open3.capture2(PS, "-ww", "-A", "-o", "pid=,pgid=,etime=,command=")
+      status.success? ? parse_group(out, pgid, now:) : []
+    rescue SystemCallError
+      []
+    end
+
+    # The Members of +pgid+ in `ps -o pid=,pgid=,etime=,command=` output.
+    def parse_group(ps_output, pgid, now:)
+      ps_output.each_line.filter_map do |line|
+        pid, group, etime, command = line.strip.split(/\s+/, 4)
+        Member.new(pid: pid.to_i, started: now - elapsed_seconds(etime), command: command.to_s) if group.to_i == pgid
+      end
+    end
+
+    # Seconds in a ps etime, "[[dd-]hh:]mm:ss" (or bare seconds).
+    def elapsed_seconds(etime)
+      days, clock = etime.include?("-") ? etime.split("-", 2) : [0, etime]
+      clock.split(":").map(&:to_i).reduce(0) { |total, part| (total * 60) + part } + (days.to_i * 86_400)
+    end
+
+    # Why +ps+ cannot answer for jobs, or nil when it can: it must accept
+    # the flags command_line uses and name the current process. BusyBox
+    # ps (Alpine) rejects -ww and -p; without this every job would look
+    # gone and a second start would run a duplicate.
+    def ps_problem(ps: PS)
+      out, err, status = Open3.capture3(ps, "-ww", "-o", "pid=,command=", "-p", Process.pid.to_s)
+      return err.lines.first.to_s.strip.then { it.empty? ? "exit status #{status.exitstatus}" : it } unless status.success?
+
+      out.include?(Process.pid.to_s) ? nil : "it printed nothing for pid #{Process.pid}"
+    rescue SystemCallError => e
+      e.message
+    end
+
+    # Raises Unusable unless +ps+ works (see ps_problem); asks once per
+    # process and ps.
+    def require_ps!(ps: PS)
+      @usable ||= {}
+      problem = @usable.fetch(ps) { @usable[ps] = ps_problem(ps:) }
+      return unless problem
+
+      raise Unusable, "#{ps} cannot list processes: #{problem}. Background jobs need a ps that accepts " \
+                      "-ww, -o and -p (procps on Linux, the system ps on macOS and BSD); on Alpine, `apk add procps`"
+    end
+
+    # Whether pid is alive and its command line carries every word. The
+    # current process counts unless +self_ok+ is false: the watcher asks
+    # about itself, but a job record naming our own pid is a reused pid.
+    # An empty word matches nothing, since every command line holds "".
+    # +command:+ supplies the command line in place of asking ps (tests,
+    # or a caller that already has it).
+    def matches?(pid, *words, command: nil, self_ok: true)
+      return true if self_ok && pid == Process.pid
+
+      command ||= command_line(pid)
+      !command.nil? && words.all? { |word| !word.to_s.empty? && command.include?(word.to_s) }
+    end
+  end
+end

data/lib/asgard/schedule/commands.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "pathname"
 require "shellwords"
 
 # warn is silent when $VERBOSE is nil and bypasses $stderr in Ruby 4.0 (see
@@ -23,9 +22,9 @@ module Asgard
       no_commands do
         def schedules = Schedule.declarations
 
-        def schedule_root = (Asgard.find_task_file&.dirname || Pathname.pwd).to_s
+        def schedule_root = @schedule_root ||= File.realpath((Asgard.find_task_file&.dirname || Pathname.pwd).to_s)
 
-        def schedule_project = File.basename(schedule_root)
+        def schedule_project = @schedule_project ||= Schedule.project_slug(schedule_root)
 
         def scheduler
           @scheduler ||= Schedule.backend_class.new(project: schedule_project, root: schedule_root)
@@ -102,7 +101,9 @@ module Asgard
           return [scheduler, name] if scheduler.installed_names.include?(name)
 
           project, entry = name.include?("/") ? name.split("/", 2) : [nil, name]
-          matches = scheduler.installed_entries.select { |slug, found| found == entry && (project.nil? || slug == project) }
+          matches = scheduler.installed_entries.select do |slug, found|
+            found == entry && (project.nil? || Schedule.project_match?(slug, project))
+          end
           abort "#{name} is in several projects (#{matches.map(&:first).join(', ')}); name one as project/#{entry}." if matches.size > 1
 
           matches.first&.then { |slug, found| [backend_for(slug), found] }
@@ -144,13 +145,15 @@ module Asgard
           by_project.to_h { |slug, entries| [project_title(slug, entries.first.last), entry_rows(entries)] }
         end
 
-        # "Project: slug (/full/path)", the path read from one of the project's job files.
+        # "Project: slug (/full/path)", the path read from one of the project's
+        # job files, flagged when the directory is gone (the project moved or
+        # was deleted, and its jobs with it should be).
         def project_title(slug, name)
           dir = slug == current_slug ? schedule_root : backend_for(slug).installed_directory(name)
-          ["Project: #{slug}", ("(#{dir})" if dir)].compact.join(" ")
+          ["Project: #{slug}", ("(#{dir})" if dir), ("[root missing]" if dir && !File.directory?(dir))].compact.join(" ")
         end
 
-        def current_slug = Asgard.find_task_file && Schedule.slug(schedule_project)
+        def current_slug = Asgard.find_task_file && schedule_project
 
         def backend_for(slug) = Schedule.backend_class.new(project: slug, root: schedule_root)
 
@@ -222,17 +225,42 @@ module Asgard
 
         path = backend.log_path(entry)
         return $stderr.puts "#{name} has no log yet (#{path}); it is created on the first run." unless File.exist?(path)
-        return sh("tail -n +1 -f #{path.shellescape}", silent: true, exec: true) if options[:follow]
+        return sh("tail -n +1 -f #{path.shellescape}", silent: true) if options[:follow]
 
         IO.copy_stream(path, $stdout)
       end
 
-      desc "remove", "Remove all of this project's installed entries"
-      def remove
+      desc "remove [NAME]", "Remove one installed entry (NAME or project/NAME), or all of this project's"
+      def remove(name = nil)
+        return remove_one(name) if name
+
         names = scheduler.installed_names
         return puts "No scheduled tasks installed for #{schedule_project}." if names.empty?
 
-        names.each { |name| schedule_uninstall(name) }
+        names.each { |entry| schedule_uninstall(entry) }
+      end
+
+      desc "prune", "Remove every project's entries whose project directory no longer exists"
+      method_option :dry_run, aliases: "-n", type: :boolean, desc: "Only list what would be removed"
+      def prune
+        orphans = Schedule.orphaned_entries(scheduler.installed_entries,
+                                            directory_of: ->(slug, name) { backend_for(slug).installed_directory(name) })
+        return puts "Nothing to prune: every installed entry's project directory exists." if orphans.empty?
+
+        orphans.each do |slug, name|
+          next puts "would remove #{slug}/#{name}" if options[:dry_run]
+
+          backend_for(slug).uninstall(name)
+          puts "removed #{slug}/#{name}"
+        end
+      end
+
+      no_commands do
+        def remove_one(name)
+          backend, entry = resolve_installed(name)
+          backend.uninstall(entry)
+          puts "removed #{name}"
+        end
       end
     end
   end