asgard 0.5.1 → 0.6.0

data/lib/asgard/state_dir.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "tempfile"
+
+module Asgard
+  # The layout every kind of per-project state shares: a directory per
+  # project under $XDG_STATE_HOME/asgard/<kind>/<project-slug>/ (by default
+  # ~/.local/state), a file in it naming the project root so a machine-wide
+  # listing can find every project without loading its .loki, and JSON
+  # records written whole then renamed. The watcher files its state under
+  # "watch" and background jobs under "jobs".
+  module StateDir
+    ROOT_FILE = "root"
+
+    # The size a log may reach before it is rotated, unless ASGARD_LOG_MAX
+    # (bytes) says otherwise.
+    LOG_MAX = 10 * 1024 * 1024
+
+    module_function
+
+    def base_dir(kind, home: Dir.home, env: ENV)
+      File.join(env["XDG_STATE_HOME"] || File.join(home, ".local", "state"), "asgard", kind.to_s)
+    end
+
+    # Every project with a state directory under base, as [slug, root] pairs.
+    def all(kind = nil, base: base_dir(kind))
+      Dir.glob(File.join(base, "*", ROOT_FILE)).map do |file|
+        [File.basename(File.dirname(file)), File.read(file).strip]
+      end
+    end
+
+    # Written whole then renamed, so a reader never sees a partial file.
+    # The tempfile is a dotfile: a "*.json" glob or a name pattern that
+    # lists records never picks up one that is still being written.
+    def write_json(path, value)
+      dir = File.dirname(path)
+      FileUtils.mkdir_p(dir)
+      Tempfile.create([".asgard", ".json"], dir) do |file|
+        file.write(JSON.pretty_generate(value))
+        file.flush
+        File.rename(file.path, path)
+      end
+    end
+
+    # The parsed record, or nil when there is none or it is not JSON (a
+    # disk that filled up, a hand edit): one bad file must not take down
+    # every listing that reads it.
+    def read_json(path)
+      return nil unless File.exist?(path)
+
+      JSON.parse(File.read(path), symbolize_names: true)
+    rescue JSON::ParserError
+      nil
+    end
+
+    # The log size limit: ASGARD_LOG_MAX when it is a positive number of
+    # bytes, else LOG_MAX.
+    def log_limit(env = ENV)
+      bytes = Integer(env["ASGARD_LOG_MAX"].to_s, exception: false)
+      bytes&.positive? ? bytes : LOG_MAX
+    end
+
+    # Moves +path+ to +path+.1 (replacing an older one) once it is larger
+    # than +max_bytes+; the new name when it did, else nil. A task still
+    # writing to the old file carries on into path.1, and the next line
+    # starts a fresh file.
+    def rotate(path, max_bytes:)
+      return nil unless File.exist?(path) && File.size(path) > max_bytes
+
+      File.rename(path, "#{path}.1")
+      "#{path}.1"
+    end
+
+    # Creates +dir+, readable by its owner only (logs and job scripts can
+    # hold secrets), and records the project root in it.
+    def record_root(dir, root)
+      FileUtils.mkdir_p(dir, mode: 0o700)
+      File.chmod(0o700, dir)
+      File.write(File.join(dir, ROOT_FILE), "#{root}\n")
+    end
+  end
+end

data/lib/asgard/tasks.rb

@@ -2,6 +2,7 @@
 
 require_relative "schedule"
 require_relative "watch"
+require_relative "jobs"
 
 # Tasks is the single conventional entry point for all .loki files.
 # It is pre-defined by the gem so .loki files never need to declare a class.
@@ -18,11 +19,13 @@ class Tasks < Asgard::Base
                type:    :boolean,
                default: false,
                desc:    "Enable debug mode ($DEBUG = true)"
+  no_negate :debug
 
   class_option :verbose,
                type:    :boolean,
                default: false,
                desc:    "Enable verbose output ($VERBOSE = true)"
+  no_negate :verbose
 
   class_option :version,
                type:    :boolean,
@@ -53,4 +56,9 @@ class Tasks < Asgard::Base
   subcommand "_watcher", Asgard::Watch::Commands
   map "watcher" => :_watcher
   Asgard::Watch::Commands.commands.each_value { |command| command.ancestor_name = "watcher" }
+
+  desc "jobs SUBCOMMAND", "List, stop and restart the background jobs started with `sh ..., background: true`"
+  subcommand "_jobs", Asgard::Jobs::Commands
+  map "jobs" => :_jobs
+  Asgard::Jobs::Commands.commands.each_value { |command| command.ancestor_name = "jobs" }
 end

data/lib/asgard/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Asgard
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/lib/asgard/watch/changes.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
+require "json"
+
 module Asgard
   # The files a watcher trigger saw change, as three lists of paths relative
   # to the project root. The watcher hands them to the task through three
   # environment variables (newline-separated, since paths may hold spaces
   # or colons); the `changes` Kernel method reads them back. Every list is
   # empty when the task is run by hand, so one task body serves both.
+  # Lists too big for the environment come in a JSON file named by
+  # ASGARD_CHANGES_FILE instead.
   class Changes < Data.define(:modified, :added, :removed)
     VARIABLES = { modified: "ASGARD_MODIFIED", added: "ASGARD_ADDED", removed: "ASGARD_REMOVED" }.freeze
+    FILE_VARIABLE = "ASGARD_CHANGES_FILE"
 
     class << self
       # The changes behind this process, read once from ENV. Tests and a
@@ -17,7 +22,20 @@ module Asgard
       def forget = @current = nil
 
       # Parses the trigger variables; a missing one is an empty list.
-      def from_env(env = ENV) = new(**VARIABLES.to_h { |member, variable| [member, env[variable].to_s.split("\n")] })
+      def from_env(env = ENV)
+        file = env[FILE_VARIABLE].to_s
+        return from_file(file) unless file.empty?
+
+        new(**VARIABLES.to_h { |member, variable| [member, env[variable].to_s.split("\n")] })
+      end
+
+      # The lists a spilled trigger wrote; empty once the file is gone.
+      def from_file(path)
+        lists = JSON.parse(File.read(path), symbolize_names: true)
+        new(**VARIABLES.keys.to_h { |member| [member, Array(lists[member])] })
+      rescue Errno::ENOENT
+        new
+      end
     end
 
     def initialize(modified: [], added: [], removed: [])
@@ -40,6 +58,13 @@ module Asgard
 
     module_function
 
+    # One Changes holding every path of +list+, each kind in first-seen
+    # order without repeats: what several triggers that waited for the same
+    # busy task hand its next run.
+    def merge_changes(list)
+      Changes.new(**Changes.members.to_h { |member| [member, list.flat_map(&member).uniq] })
+    end
+
     # A listener batch ({ modified:, added:, removed: }) narrowed to the
     # kinds in events (a Set from normalize_events), as a Changes; nil
     # when nothing is left, so the dispatcher skips the trigger.

data/lib/asgard/watch/commands.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "pathname"
 require "shellwords"
 require "time"
 
@@ -30,14 +29,25 @@ module Asgard
         end
 
         # The LAST RUN column from an entry's record: never, still running,
-        # or its exit status with duration and finish time.
-        def last_run(entry)
+        # did not finish (no exit was recorded and its process is gone: a
+        # crash, a kill -9), or its exit status with duration and finish
+        # time. +running+ says whether an unfinished entry's task still runs.
+        def last_run(entry, running: State.method(:running?))
           return "never" unless entry
-          return "running (pid #{entry[:pid]})" unless entry.key?(:finished)
+          return "#{running.call(entry) ? 'running' : 'did not finish'} (pid #{entry[:pid]})" unless entry.key?(:finished)
 
           finished = Time.iso8601(entry[:finished]).strftime("%Y-%m-%d %H:%M:%S")
           "#{entry[:status]} (#{format('%.1fs', entry[:seconds])}, #{finished})"
         end
+
+        # How `start` launches `watcher run`: in the project root, its own
+        # process group, output to the log, and no stdin, which it would
+        # otherwise share with the terminal that ran `start`.
+        def daemon_spawn_options(root, log) = { chdir: root, pgroup: true, in: File::NULL, out: [log, "a"], err: %i[child out] }
+
+        # The shell command `log -f` runs. -F, not -f: it keeps following
+        # watcher.log when rotation renames it and a new one starts.
+        def follow_command(path) = "tail -n +1 -F #{path.shellescape}"
       end
 
       desc "tree", "Print a tree of the watcher subcommands", hide: true
@@ -49,7 +59,7 @@ module Asgard
 
         def watch_root = @watch_root ||= File.realpath((Asgard.find_task_file&.dirname || Pathname.pwd).to_s)
 
-        def watch_project = File.basename(watch_root)
+        def watch_project = @watch_project ||= Schedule.project_slug(watch_root)
 
         def state = @state ||= State.new(project: watch_project, root: watch_root)
 
@@ -95,35 +105,64 @@ module Asgard
       def daemon
         specs = declared_watches
         refuse_if_another_running
-        config = Daemon::Config.new(declarations: specs, root: watch_root, state:, context: spawn_context, tasks: Tasks.all_commands.keys)
+        config = Daemon::Config.new(declarations: specs, root: watch_root, state:, context: spawn_context, tasks: Tasks.all_commands.keys,
+                                    pending: ENV.delete(State::PENDING_ENV))
         Daemon.new(config).run
       end
 
+      # What a reload runs before it re-execs: the checks `run` makes,
+      # without running anything. Exits non-zero, saying why on stderr,
+      # when the task files do not load, nothing is watched, or a watch
+      # names a task the project does not define.
+      desc "check", "Exit 0 when the watches would run (what a reload checks first)", hide: true
+      def check
+        declared_watches
+        nil
+      end
+
       desc "start", "Start the watcher for this project in the background"
       def start
         specs = declared_watches
         refuse_if_running
         log = state.log_path
         state.prepare.write_declarations(specs.values)
-        pid = Process.spawn(watch_asgard, "watcher", "run", chdir: watch_root, pgroup: true, out: [log, "a"], err: %i[child out])
+        lock = state.acquire_lock or abort("watcher is already running for #{watch_project}; `asgard watcher stop` first.")
+        pid  = Process.spawn({ State::LOCK_ENV => lock.fileno.to_s }, watch_asgard, "watcher", "run",
+                             **Commands.daemon_spawn_options(watch_root, log), lock.fileno => lock.fileno)
+        lock.close # the daemon holds it now
         Process.detach(pid)
         state.write_pid(pid)
         puts "started watcher for #{watch_project} (pid #{pid}); log: #{log}"
       end
 
-      desc "stop", "Stop the watcher; a task already running finishes first"
+      desc "stop", "Stop the watcher; a task already running finishes first (--force: end running tasks)"
+      method_option :force, aliases: "-f", type: :boolean, desc: "End running tasks (TERM, then KILL) instead of waiting for them"
       def stop
         pid = state.running_pid or abort("watcher is not running for #{watch_project}.")
         Process.kill("TERM", pid)
-        while Reporter.alive?(pid)
+        waited = 0
+        while State.daemon?(pid)
           running = state.running_entries.map { |entry| "#{entry[:entry]} (pid #{entry[:pid]})" }
           puts "watcher: waiting for #{running.join(', ')}" unless running.empty?
           sleep STOP_WAIT
+          force_stop(pid) if options[:force] && (waited += 1) == 1
         end
         state.clear
         puts "stopped watcher for #{watch_project}"
       end
 
+      no_commands do
+        # The second TERM, a moment after the first so the two are not
+        # merged into one pending signal: the daemon takes it as "end the
+        # running tasks", which it does only for groups it can prove are
+        # its own tasks.
+        def force_stop(pid)
+          Process.kill("TERM", pid) if State.daemon?(pid)
+        rescue Errno::ESRCH
+          nil
+        end
+      end
+
       no_commands do
         # True when list should cover every project: asked for with --all, or
         # there is no .loki here, so no single project to scope to.
@@ -151,7 +190,8 @@ module Asgard
           State.all.to_h do |slug, root|
             other = State.new(project: slug, root:)
             own   = Asgard.find_task_file && root == watch_root
-            ["Project: #{slug} (#{root})", Commands.project_rows(other, own ? declared_records : other.read_declarations)]
+            title = ["Project: #{slug} (#{root})", ("[root missing]" unless File.directory?(root))].compact.join(" ")
+            [title, Commands.project_rows(other, own ? declared_records : other.read_declarations)]
           end
         end
       end
@@ -197,11 +237,11 @@ module Asgard
       end
 
       desc "log", "Print this project's watcher log to STDOUT"
-      method_option :follow, aliases: "-f", type: :boolean, desc: "Keep printing new output as it arrives (tail -f)"
+      method_option :follow, aliases: "-f", type: :boolean, desc: "Keep printing new output as it arrives, across rotation (tail -F)"
       def log
         path = state.log_path
         return $stderr.puts "#{watch_project} has no watcher log yet (#{path}); it is created on the first start." unless File.exist?(path)
-        return sh("tail -n +1 -f #{path.shellescape}", silent: true, exec: true) if options[:follow]
+        return sh(Commands.follow_command(path), silent: true) if options[:follow]
 
         IO.copy_stream(path, $stdout)
       end