asgard 0.4.1 → 0.5.0

data/lib/asgard/watch/dispatcher.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "ractor_queue"
+
+module Asgard
+  module Watch
+    # Routes listener batches to worker queues. Built once at startup from
+    # the declarations: a dispatch table from listener id (the entry name)
+    # to its spec, and one worker per distinct task, so two declarations
+    # for the same task share a worker and never overlap. Nothing here is
+    # mutated after construction, and the loop never blocks on a worker.
+    class Dispatcher
+      # Triggers a busy task can have waiting. RactorQueue rounds this up.
+      QUEUE_CAPACITY = 256
+
+      attr_reader :dispatch, :workers
+
+      # start: makes a worker from (queue, reports); the default starts a
+      # real Ractor. clock: supplies the trigger timestamp.
+      def initialize(declarations, context, reports:, start: Worker.method(:start), clock: Time)
+        specs     = declarations.values
+        @context  = context
+        @reports  = reports
+        @clock    = clock
+        @dispatch = specs.to_h { |spec| [spec[:name], spec] }.freeze
+        @workers  = specs.map { |spec| spec[:task] }.uniq.to_h do |task|
+          queue = RactorQueue.new(capacity: QUEUE_CAPACITY)
+          [task, { queue:, ractor: start.call(queue, reports) }.freeze]
+        end.freeze
+      end
+
+      # Pops [listener id, batch] pairs from inbox until it pops nil.
+      def run(inbox)
+        while (id, batch = inbox.pop)
+          route(id, batch)
+        end
+      end
+
+      def start(inbox) = Thread.new { run(inbox) }
+
+      # One batch: drop the kinds the entry does not watch, build the frozen
+      # description, report the trigger, hand it to the task's worker. The
+      # description, or nil when nothing was left to run.
+      def route(id, batch)
+        spec    = @dispatch.fetch(id)
+        changes = Watch.filter_events(batch, spec[:events]) or return nil
+        task    = spec[:task]
+        description = Spawn.description(spec, changes, @context)
+        @reports.push(Ractor.make_shareable({ kind: :trigger, entry: spec[:name], task:, at: @clock.now.to_f, changes: changes.to_h }))
+        @workers.fetch(task)[:queue].push(description)
+        description
+      end
+    end
+  end
+end

data/lib/asgard/watch/reporter.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Asgard
+  module Watch
+    # The one writer of the watcher's log and per-entry state. Workers and
+    # the dispatcher push frozen reports onto a shared queue; the reporter
+    # drains it in a main-Ractor thread, appends a log line per report,
+    # keeps the latest record per entry on disk through the state, and
+    # tracks which pids are in flight so shutdown can wait for them.
+    class Reporter
+      class << self
+        # One log line for a report. behind: is the pid a trigger is queued
+        # behind when its task is still running.
+        def log_line(report, behind: nil)
+          kind, entry, task, pid, at = report.values_at(:kind, :entry, :task, :pid, :at)
+          head    = "#{Time.at(at).strftime('%Y-%m-%d %H:%M:%S')} #{kind} #{entry} task=#{task}"
+          changed = paths(report)
+          case kind
+          when :trigger then "#{head} #{changed}#{" (queued behind pid #{behind})" if behind}"
+          when :spawn   then "#{head} pid=#{pid} #{changed}"
+          else               "#{head} pid=#{pid} status=#{report[:status]} seconds=#{format('%.2f', report[:seconds])}"
+          end
+        end
+
+        # The entry's record after this report: a spawn starts a new record
+        # (no finished/status yet, which is how `list` shows "running"), an
+        # exit completes the previous one.
+        def entry_record(report, previous = nil)
+          entry, task, pid, at = report.values_at(:entry, :task, :pid, :at)
+          stamp = Time.at(at).iso8601
+          return { entry:, task:, pid:, started: stamp, changes: report[:changes] } if report[:kind] == :spawn
+
+          { entry:, task:, pid:, **previous.to_h, finished: stamp, status: report[:status], seconds: report[:seconds] }
+        end
+
+        # The changed paths of a trigger or spawn report, space-separated; "" for an exit.
+        def paths(report) = report.fetch(:changes) { {} }.values.flatten.join(" ")
+
+        # Whether a process exists; one we may not signal still exists.
+        def alive?(pid)
+          Process.kill(0, pid)
+          true
+        rescue Errno::ESRCH
+          false
+        rescue Errno::EPERM
+          true
+        end
+      end
+
+      # state: responds to write_entry(name, record).
+      def initialize(reports, log:, state:)
+        @reports = reports
+        @log     = log
+        @state   = state
+        @records = {}
+        @running = {} # pid => task
+        @mutex   = Mutex.new
+      end
+
+      # Handles reports until it pops nil.
+      def run
+        while (report = @reports.pop)
+          handle(report)
+        end
+      end
+
+      def start = Thread.new { run }
+
+      def handle(report)
+        trigger = report[:kind] == :trigger
+        line    = Reporter.log_line(report, behind: trigger ? running_for(report[:task]) : nil)
+        File.open(@log, "a") { |file| file.puts(line) }
+        track(report) unless trigger
+      end
+
+      # Pids of tasks in flight, ascending.
+      def running = @mutex.synchronize { @running.keys.sort }
+
+      # The in-flight pid of task, or nil.
+      def running_for(task) = @mutex.synchronize { @running.key(task) }
+
+      # Blocks until no task is in flight, saying what it waits for once
+      # per interval. A pid that is gone without an exit report (a worker
+      # died between its reports) is forgotten rather than waited on.
+      def drain(interval: 1.0, out: $stdout)
+        until (pids = running).empty?
+          out.puts "watcher: waiting for pid #{pids.join(', ')}"
+          sleep interval
+          forget_dead
+        end
+      end
+
+      private
+
+      def track(report)
+        entry, pid = report.values_at(:entry, :pid)
+        record = Reporter.entry_record(report, @records[entry])
+        @records[entry] = record
+        @state.write_entry(entry, record)
+        @mutex.synchronize { record.key?(:finished) ? @running.delete(pid) : @running[pid] = record[:task] }
+      end
+
+      def forget_dead
+        @mutex.synchronize { @running.delete_if { |pid, _task| !Reporter.alive?(pid) } }
+      end
+    end
+  end
+end

data/lib/asgard/watch/spawn.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "../schedule/declaration"
+
+module Asgard
+  module Watch
+    # Builds the description of one task run: the argv, the complete
+    # environment, and where it runs and logs. The dispatcher builds it in
+    # the main Ractor, where ENV and the declarations live, and freezes it
+    # with Ractor.make_shareable; the worker Ractor that spawns the task
+    # sees nothing else, which is what keeps the daemon's own environment
+    # out of the task. Everything here is pure.
+    module Spawn
+      # The only variables a task inherits from the daemon. Everything else
+      # it sees comes from the declaration's env:, from .env and .envrc
+      # loaded in the task's own process, or from the trigger itself.
+      BASE_ENV_KEYS    = %w[PATH HOME USER LOGNAME SHELL LANG TMPDIR SSH_AUTH_SOCK].freeze
+      BASE_ENV_PATTERN = /\ALC_/
+
+      # What every trigger of one daemon run shares: the real project root,
+      # the log file tasks append to, the base environment captured at
+      # start, the asgard executable, and direnv when it is on PATH.
+      Context = Data.define(:root, :log, :base, :asgard, :direnv)
+
+      module_function
+
+      # The whitelisted variables of env (ENV by default).
+      def base_env(env = ENV) = env.select { |key, _value| BASE_ENV_KEYS.include?(key) || key.match?(BASE_ENV_PATTERN) }
+
+      # The task's full environment, lowest precedence first: the context's
+      # base, the declaration's env:, the trigger variables.
+      def environment(spec, changes, context)
+        context.base.merge(spec[:env], { "ASGARD_WATCH" => spec[:name], "ASGARD_WATCH_ROOT" => context.root }, changes.to_env)
+      end
+
+      # The task's argv: the context's asgard executable by absolute path, so
+      # a trigger runs the very asgard that started the watcher whatever PATH
+      # says, wrapped in direnv exec when available so .envrc loads in the
+      # task's process.
+      def argv(spec, context)
+        direnv  = context.direnv
+        wrapper = direnv ? [direnv, "exec", context.root] : []
+        [*wrapper, context.asgard, spec[:task]]
+      end
+
+      # The asgard executable to run: the one running now when it is an
+      # executable file named asgard (bin/asgard in a checkout, a gem
+      # binstub when installed), else the first on path. A PATH scan could
+      # find a different installed version than the one that started the
+      # watcher.
+      def executable(program = $PROGRAM_NAME, path = ENV.fetch("PATH", nil))
+        running = File.expand_path(program)
+        return running if File.basename(running) == "asgard" && File.file?(running) && File.executable?(running)
+
+        Schedule.which("asgard", path)
+      end
+
+      # Everything a worker needs to run one trigger, deep-frozen and
+      # Ractor-shareable: Strings, Arrays and Hashes only.
+      def description(spec, changes, context)
+        Ractor.make_shareable(
+          {
+            entry:   spec[:name],
+            task:    spec[:task],
+            argv:    argv(spec, context),
+            env:     environment(spec, changes, context),
+            root:    context.root,
+            log:     context.log,
+            changes: changes.to_h
+          }
+        )
+      end
+    end
+  end
+end

data/lib/asgard/watch/state.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "tempfile"
+
+module Asgard
+  module Watch
+    # Where one project's watcher keeps what outlives a daemon run: the
+    # pidfile, the log, the declarations recorded at start, and one JSON
+    # record per entry with its last run. Everything lives outside the
+    # repository under $XDG_STATE_HOME/asgard/watch/<project-slug>/ (by
+    # default ~/.local/state), with a file naming the project root so
+    # `watcher list --all` can find every project without loading it.
+    class State
+      ROOT_FILE = "root"
+
+      attr_reader :dir, :root
+
+      class << self
+        def base_dir(home: Dir.home, env: ENV)
+          File.join(env["XDG_STATE_HOME"] || File.join(home, ".local", "state"), "asgard", "watch")
+        end
+
+        # Every project with a state directory under base, as [slug, root] pairs.
+        def all(base: base_dir)
+          Dir.glob(File.join(base, "*", ROOT_FILE)).map do |file|
+            [File.basename(File.dirname(file)), File.read(file).strip]
+          end
+        end
+
+        # What `list` needs of a declaration, with the events Set as an Array.
+        def declaration_record(spec) = spec.slice(:name, :task, :paths).merge(events: spec[:events].to_a)
+
+        # Written whole then renamed, so a reader never sees a partial file.
+        def write_json(path, value)
+          dir = File.dirname(path)
+          FileUtils.mkdir_p(dir)
+          Tempfile.create(["asgard", ".json"], dir) do |file|
+            file.write(JSON.pretty_generate(value))
+            file.flush
+            File.rename(file.path, path)
+          end
+        end
+
+        def read_json(path) = File.exist?(path) ? JSON.parse(File.read(path), symbolize_names: true) : nil
+
+        # Whether an entry record describes a task still running: no exit
+        # recorded and its pid alive.
+        def running?(entry) = !entry.key?(:finished) && Reporter.alive?(entry[:pid])
+      end
+
+      # base: is the directory every project's state lives under.
+      def initialize(project:, root:, base: State.base_dir)
+        @root = root
+        @dir  = File.join(base, Schedule.slug(project))
+      end
+
+      def pid_path          = File.join(dir, "watcher.pid")
+      def log_path          = File.join(dir, "watcher.log")
+      def root_path         = File.join(dir, ROOT_FILE)
+      def declarations_path = File.join(dir, "declarations.json")
+      def entries_dir       = File.join(dir, "entries")
+      def entry_path(name)  = File.join(entries_dir, "#{name}.json")
+
+      # Creates the directories and records the root.
+      def prepare
+        FileUtils.mkdir_p(entries_dir)
+        File.write(root_path, "#{root}\n")
+        self
+      end
+
+      def write_pid(pid) = File.write(pid_path, "#{pid}\n")
+
+      def read_pid = File.exist?(pid_path) ? File.read(pid_path).to_i : nil
+
+      # The daemon's pid when one is running, else nil. A pidfile left by a
+      # crash counts as not running.
+      def running_pid
+        pid = read_pid
+        pid if pid&.positive? && Reporter.alive?(pid)
+      end
+
+      def clear = FileUtils.rm_f(pid_path)
+
+      # The declarations at start time, trimmed to what `list` shows.
+      def write_declarations(specs)
+        State.write_json(declarations_path, specs.map { |spec| State.declaration_record(spec) })
+      end
+
+      def read_declarations = State.read_json(declarations_path) || []
+
+      def write_entry(name, record) = State.write_json(entry_path(name), record)
+
+      def read_entry(name) = State.read_json(entry_path(name))
+
+      # Every entry's latest record, keyed by entry name.
+      def entries = Dir.glob(File.join(entries_dir, "*.json")).to_h { |file| [File.basename(file, ".json"), State.read_json(file)] }
+
+      # The records of tasks still running.
+      def running_entries = entries.values.select { |entry| State.running?(entry) }
+    end
+  end
+end

data/lib/asgard/watch/worker.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Asgard
+  module Watch
+    # Runs one task's triggers, one at a time, in the order they arrive on
+    # its queue. One worker exists per distinct task, each in its own
+    # Ractor, so different tasks run concurrently while a task never
+    # overlaps itself: the wait for the child is the mutex. A worker sees
+    # nothing but frozen spawn descriptions and pushes a frozen report on
+    # every spawn and every exit; it writes no files itself.
+    class Worker
+      # Starts and reaps a task from a spawn description. Kept apart from
+      # the loop so tests can stand in a recorder.
+      module Spawner
+        module_function
+
+        # The child's pid. The task runs in the project root with exactly
+        # the description's environment, in its own process group, with
+        # its output appended to the log.
+        def spawn(description)
+          Process.spawn(description[:env], *description[:argv],
+                        chdir: description[:root], pgroup: true, unsetenv_others: true,
+                        out: [description[:log], "a"], err: %i[child out])
+        end
+
+        # Blocks until the child exits; its exit code.
+        def wait(pid)
+          _, status = Process.wait2(pid)
+          exit_code(status)
+        end
+
+        # A signalled child reports 128 + the signal number, as shells do.
+        def exit_code(status) = status.exitstatus || (128 + status.termsig)
+      end
+
+      # The worker loop in its own Ractor. The Ractor is handed the two
+      # queues and the spawner (all shareable) and nothing else.
+      def self.start(queue, reports, spawner: Spawner)
+        Ractor.new(queue, reports, spawner) { |inbox, outbox, runner| Asgard::Watch::Worker.new(inbox, outbox, spawner: runner).run }
+      end
+
+      def initialize(queue, reports, spawner: Spawner, clock: Process)
+        @queue   = queue
+        @reports = reports
+        @spawner = spawner
+        @clock   = clock
+      end
+
+      # Pops descriptions and runs each to completion. A nil on the queue
+      # ends the loop.
+      def run
+        while (description = @queue.pop)
+          run_one(description)
+        end
+      end
+
+      def run_one(description)
+        entry, task = description.values_at(:entry, :task)
+        pid = @spawner.spawn(description)
+        started_at, started = stamp
+        report(kind: :spawn, entry:, task:, pid:, at: started_at, changes: description[:changes])
+        status = @spawner.wait(pid)
+        finished_at, finished = stamp
+        report(kind: :exit, entry:, task:, pid:, at: finished_at, status:, seconds: finished - started)
+      end
+
+      private
+
+      # [wall-clock seconds since the epoch, monotonic seconds]
+      def stamp = [@clock.clock_gettime(Process::CLOCK_REALTIME), @clock.clock_gettime(Process::CLOCK_MONOTONIC)]
+
+      def report(**fields) = @reports.push(Ractor.make_shareable(fields))
+    end
+  end
+end

data/lib/asgard/watch.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "watch/declaration"
+require_relative "watch/changes"
+require_relative "watch/spawn"
+require_relative "watch/worker"
+require_relative "watch/reporter"
+require_relative "watch/dispatcher"
+require_relative "watch/state"
+require_relative "watch/daemon"
+require_relative "watch/commands"
+
+module Asgard
+  # Runs asgard tasks when watched files change. Declare entries at class
+  # level in a .loki file:
+  #
+  #   class Tasks
+  #     watch :lint, paths: "lib/**/*.rb"
+  #     watch :test, paths: %w[lib test], only: /\.rb\z/
+  #   end
+  #
+  # then manage the watcher with `asgard watcher run|start|stop|list|...`.
+  module Watch
+    # A watcher could not run: an unknown task, a daemon already running.
+    class Error < Asgard::Error; end
+
+    class << self
+      # Declared entries for this run, keyed by entry name.
+      def declarations
+        @declarations ||= {}
+      end
+
+      # Records one declaration. Redeclaring an identical entry is a no-op
+      # (a .loki file loaded twice); a different entry under the same name
+      # is an error.
+      def declare(task, **settings)
+        spec  = normalize(task, **settings)
+        name  = spec[:name]
+        taken = declarations[name]
+        raise ArgumentError, "watch: #{name.inspect} is already declared; give one a distinct as: name" if taken && taken != spec
+
+        declarations[name] = spec
+      end
+    end
+
+    # The class-level `watch` declaration helper for Tasks.
+    module DSL
+      def watch(task, **) = Watch.declare(task, **)
+    end
+  end
+end

data/lib/asgard.rb

@@ -15,9 +15,9 @@ module Asgard
     loki_up
   end
 
-  # `asgard schedule list` reports every project's scheduled entries, so it
-  # works from a directory with no .loki above it.
-  def self.machine_wide?(argv) = argv.first(2) == %w[schedule list]
+  # `asgard schedule list` and `asgard watcher list` report every project's
+  # entries, so they work from a directory with no .loki above it.
+  def self.machine_wide?(argv) = [%w[schedule list], %w[watcher list]].include?(argv.first(2))
 
   # Main entry point invoked by the asgard executable.
   def self.run!(argv)

data/mkdocs.yml

@@ -159,6 +159,7 @@ nav:
       - Shell Helpers: shell.md
       - Environment: environment.md
       - Scheduled Tasks: schedule.md
+      - Watched Files: watch.md
   - Task Files: task-files.md
   - API Reference: api.md
   - Examples: examples.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: asgard
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Dewayne VanHoozer
@@ -51,6 +51,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: ractor_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
 description: |
   A powerful Ruby-based task runner for any kind of project with task dependency tracking
   and concurrent execution of designated tasks. Uses Thor for its rich CLI options, var
@@ -95,7 +123,9 @@ files:
 - docs/task-files.md
 - docs/tasks.md
 - docs/variables.md
+- docs/watch.md
 - examples/.env
+- examples/.envrc
 - examples/.loki
 - examples/bad.loki
 - examples/concurrent.loki
@@ -108,6 +138,14 @@ files:
 - examples/subdir/.loki
 - examples/subdir/import_demo.loki
 - examples/subdir/import_up_demo.loki
+- examples/watcher_examples/.envrc
+- examples/watcher_examples/.loki
+- examples/watcher_examples/README.md
+- examples/watcher_examples/bin/hello
+- examples/watcher_examples/data/records.json
+- examples/watcher_examples/docs/drafts/ideas.md
+- examples/watcher_examples/docs/index.md
+- examples/watcher_examples/lib/greeter.rb
 - gem_tasks.loki
 - git.loki
 - lib/asgard.rb
@@ -130,6 +168,16 @@ files:
 - lib/asgard/shell.rb
 - lib/asgard/tasks.rb
 - lib/asgard/version.rb
+- lib/asgard/watch.rb
+- lib/asgard/watch/changes.rb
+- lib/asgard/watch/commands.rb
+- lib/asgard/watch/daemon.rb
+- lib/asgard/watch/declaration.rb
+- lib/asgard/watch/dispatcher.rb
+- lib/asgard/watch/reporter.rb
+- lib/asgard/watch/spawn.rb
+- lib/asgard/watch/state.rb
+- lib/asgard/watch/worker.rb
 - mkdocs.yml
 - quality.loki
 - quality_rails.loki
@@ -157,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.22
+rubygems_version: 4.0.23
 specification_version: 4
 summary: A powerful Ruby-based task runner
 test_files: []