asgard 0.5.1 → 0.6.0

data/lib/asgard/base/dispatch.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "English"
+
 module Asgard
   class Base < Thor
     # Task execution engine: resolves and runs declared dependencies (in
@@ -15,6 +17,69 @@ module Asgard
         base.extend(ClassMethods)
       end
 
+      # A finished task's outcome: :failed when its run ended in an error.
+      def self.run_state(target, done:, failed:)
+        return :failed if failed.key?(target)
+
+        done.include?(target) ? :done : :free
+      end
+
+      # A group's errors without the ones that only echo another failure:
+      # a thread that waited on a failed shared dependency. When nothing
+      # else is left, the echoes are all there is to report.
+      def self.primary_failures(errors)
+        primary = errors.reject { |_, error| error.is_a?(DepFailed) }
+        primary.empty? ? errors : primary
+      end
+
+      # A task that is already running further up +chain+ (the tasks this
+      # thread is inside, outermost first) and is asked for again can never
+      # finish: `def a = invoke(:b)` with `def b = invoke(:a)`. It would
+      # wait on itself forever, so name the cycle instead.
+      #
+      # Each entry, +target+ included, is [class, task name]: a task of the
+      # same name in another class is a different task. A subcommand
+      # group's `build` runs inside the `build` that dispatches to it, and
+      # `jobs restart` runs a project task that may itself be `restart`.
+      def self.check_cycle!(chain, target)
+        start = chain.index(target)
+        return unless start
+
+        raise Asgard::CircularDependencyError, [*chain[start..], target].map(&:last).join(" -> ")
+      end
+
+      # The options a dependency runs with. They start as the typed task's
+      # (+typed+), so an option the dependency does not declare passes
+      # through unchanged. The dependency's own defaults (+declared+, class
+      # options included) apply over those. Last, each option the user
+      # actually gave (+raw+, read against +typed_declared+, the typed
+      # task's declarations) reaches the dependency when it declares an
+      # option of that name. Words are read only as the typed task means
+      # them: a short alias that names something else in the dependency
+      # (-f for --file, not --force) never reaches it. A required option
+      # of the dependency is not enforced, as it was not before.
+      def self.dep_options(declared, typed_declared, raw, typed)
+        defaults = Thor::Options.new(declared, {}, false, true).parse([])
+        given    = given_options(typed_declared, raw).slice(*declared.values.map(&:human_name))
+        typed.merge(defaults).merge(given).freeze
+      end
+
+      # Only the options +raw+ spells out, by name, with no defaults filled
+      # in: an Array of command-line words read against +declared+, or the
+      # Hash a caller passed to new.
+      def self.given_options(declared, raw)
+        return raw.to_h.transform_keys(&:to_s) unless raw.is_a?(Array)
+
+        bare = declared.transform_values { without_default(it) }
+        Thor::Options.new(bare, {}, false, true).parse(raw).to_h
+      end
+
+      # A copy of +option+ that parses the same words but has no default.
+      def self.without_default(option)
+        Thor::Option.new(option.name, type: option.type, aliases: option.aliases, enum: option.enum,
+                                      repeatable: option.repeatable, lazy_default: option.lazy_default)
+      end
+
       module ClassMethods
         def _running
           @_running ||= Set.new
@@ -28,6 +93,13 @@ module Asgard
           @_cond ||= Hash.new { |h, k| h[k] = ConditionVariable.new }
         end
 
+        # Tasks whose body raised or exited, keyed by name, with the error
+        # that ended them: a thread that waited on one re-raises instead of
+        # running its own body on a prerequisite that never finished.
+        def _failed
+          @_failed ||= {}
+        end
+
         # Completed tasks' return values, keyed by task name — lets a task
         # already run by one thread hand its result to another thread that
         # shares the dependency, without either touching `self`.
@@ -46,6 +118,7 @@ module Asgard
             @_done    = Set.new
             @_cond    = Hash.new { |h, k| h[k] = ConditionVariable.new }
             @_results = {}
+            @_failed  = {}
           end
         end
       end
@@ -56,14 +129,26 @@ module Asgard
         $DEBUG   = true if options[:debug]
         $VERBOSE = true if options[:verbose]
         target = command.name.to_sym
-        return cached_result(target) unless acquire_run_token(target)
+        Dispatch.check_cycle!(_task_frames.map { [it.owner, it.task] }, [self.class, target])
+        case _acquire_run_token(target)
+        when :done   then return _cached_result(target)
+        when :failed then raise DepFailed.new(target, self.class._failed[target])
+        end
 
         result = nil
+        error  = nil
         begin
-          resolved_deps = run_deps_for(target)
-          result = with_dep_results(resolved_deps) { command.run(self, *) }
+          result = _in_task_frame(target) do
+            resolved_deps = _run_deps_for(target)
+            _with_dep_results(resolved_deps) { command.run(self, *) }
+          end
+        rescue Exception => e # rubocop:disable Lint/RescueException -- an exit or interrupt ends the task too
+          # Captured here, not read from $! in an ensure: $! still holds a
+          # caller's exception when this runs inside its rescue block.
+          error = e
+          raise
         ensure
-          signal_done(target, result)
+          _signal_done(target, result, error)
         end
         result
       end
@@ -78,9 +163,41 @@ module Asgard
         dep_results[task.to_sym]
       end
 
+      # One running task, from its dependencies through its body: the class
+      # it belongs to, its name and how many background jobs it has started
+      # so far. Frames live on
+      # a thread-local stack, so a task running as a dependency in a
+      # parallel stage sees itself, not the task the user typed. A parallel
+      # stage's thread starts with a copy of its parent's stack, which is
+      # how a cycle through that stage is still seen (see check_cycle!).
+      TaskFrame = Struct.new(:owner, :task, :jobs)
+
+      # The task whose body is running on this thread, or nil outside one
+      # (a helper called at .loki top level).
+      def current_task = _task_frames.last&.task
+
+      # The next sequence number for a background job of the current task:
+      # 1 on the first call of each run of the body, then 2, 3, ...
+      def _next_job_sequence
+        frame = _task_frames.last
+        raise Asgard::Error, "no task is running on this thread" unless frame
+
+        frame.jobs += 1
+      end
+
       private
 
-      def with_dep_results(results)
+      def _task_frames = Thread.current[:asgard_task_frames] ||= []
+
+      def _in_task_frame(task)
+        frames = _task_frames
+        frames.push(TaskFrame.new(self.class, task, 0))
+        yield
+      ensure
+        frames.pop
+      end
+
+      def _with_dep_results(results)
         thread   = Thread.current
         previous = thread[:asgard_dep_results]
         thread[:asgard_dep_results] = results
@@ -89,71 +206,180 @@ module Asgard
         thread[:asgard_dep_results] = previous
       end
 
-      def cached_result(target)
+      def _cached_result(target)
         klass = self.class
         klass._ran_mutex.synchronize { klass._results[target] }
       end
 
-      def acquire_run_token(target)
+      # :run when this thread is to run +target+, :done when another
+      # already ran it, :failed when that run raised or exited. A thread
+      # that finds it running waits for the outcome.
+      def _acquire_run_token(target)
         klass   = self.class
         mutex   = klass._ran_mutex
         done    = klass._done
         running = klass._running
 
         mutex.synchronize do
-          if done.include?(target)
-            false
-          elsif running.include?(target)
+          if running.include?(target) && !done.include?(target)
             klass._cond[target].wait(mutex) until done.include?(target)
-            false
-          else
-            running.add(target)
-            true
           end
+          next Dispatch.run_state(target, done:, failed: klass._failed) if done.include?(target)
+
+          running.add(target)
+          :run
         end
       end
 
-      def run_deps_for(target)
+      def _run_deps_for(target)
         stages = self.class._deps[target]
         return {} unless stages&.any?
 
-        stages.each_with_object({}) { |group, acc| acc.merge!(run_dep_group(group)) }
+        stages.each_with_object({}) { |group, acc| acc.merge!(_run_dep_group(group)) }
       end
 
-      def run_dep_group(group)
-        return { group.first => run_dep(group.first) } unless group.size > 1
+      def _run_dep_group(group)
+        return { group.first => _run_dep(group.first) } unless group.size > 1
 
-        threads = group.map { |task| [task, Thread.new { run_dep(task) }] }
-        errors  = []
+        threads = group.map { |task| [task, _dep_thread(task)] }
+        errors  = {}
         results = {}
+        # Join every thread before raising, so one failure never ends the
+        # process while the other tasks' subprocesses run on, orphaned.
         threads.each do |task, t|
           results[task] = t.value
-        rescue => e
-          errors << e
+        rescue StandardError, ScriptError => e
+          errors[task] = e
         end
 
-        if errors.size == 1
-          raise errors.first
-        elsif errors.any?
-          errors.each { |e| warn "asgard: #{e.message}" }
-          raise Asgard::Error, "#{errors.size} parallel dependencies failed"
-        end
+        errors = Dispatch.primary_failures(errors)
+        _raise_dep_failure(errors.values.first) if errors.size == 1
+        _raise_group_failure(errors) if errors.any?
 
         results
       end
 
-      def signal_done(target, result)
+      # A SystemExit that ended a dependency thread (a failing `sh`, an
+      # `abort`), carried out of the thread as an ordinary error. Ruby
+      # re-raises an unhandled SystemExit from any thread in the main
+      # thread, which would end the process in the middle of the join.
+      class DepExit < StandardError
+        attr_reader :status
+
+        def initialize(status)
+          @status = status
+          super("exit #{status}")
+        end
+      end
+
+      # Raised in a thread that waited on +task+ when +task+ failed. Its
+      # +error+ is the failure itself, which some other thread reports, so
+      # the joiner counts only that one (see primary_failures).
+      class DepFailed < StandardError
+        attr_reader :task, :error
+
+        def initialize(task, error)
+          @task  = task
+          @error = error
+          super("dependency '#{task}' failed")
+        end
+      end
+
+      # Ruby's own report would print the full backtrace to stderr before
+      # the error reaches asgard's handler; the joiner re-raises it anyway.
+      # Each parallel task runs on its own instance (same options), so two
+      # tasks in one stage never race on an instance variable; results
+      # pass between tasks through dep_result, not through self.
+      def _dep_thread(task)
+        instance = _parallel_instance
+        frames   = _task_frames.dup
+        Thread.new do
+          thread = Thread.current
+          thread.report_on_exception = false
+          thread[:asgard_task_frames] = frames
+          instance.__send__(:_run_dep, task)
+        rescue SystemExit => e
+          raise DepExit, e.status
+        end
+      end
+
+      # Built as Thor built this instance: the same command line and the
+      # same config, which names the typed command. Thor then reads the
+      # words against that command's options, as it did the first time,
+      # so check_unknown_options! knows them. The task the copy runs reads
+      # its own options from the same words (see dep_options).
+      def _parallel_instance
+        config   = _initializer_config
+        typed    = config[:current_command]
+        config   = config.merge(command_options: typed.options) if typed
+        instance = self.class.new([], _raw_options, config)
+        instance.options = options
+        instance
+      end
+
+      # The options this instance was created with, before parsing: the
+      # command line's words, or the Hash a caller passed to new.
+      def _raw_options = @_initializer[1]
+
+      # The config Thor created this instance with; its :current_command
+      # is the task that was typed (or invoked).
+      def _initializer_config = @_initializer[2] || {}
+
+      # What the command line's words are read against: the class options
+      # plus the typed task's own.
+      def _typed_declared
+        typed = _initializer_config[:current_command]
+        declared = self.class.class_options
+        typed ? declared.merge(typed.options) : declared
+      end
+
+      # Runs the block with +opts+ as this instance's options.
+      def _with_options(opts)
+        previous     = options
+        self.options = opts
+        yield
+      ensure
+        self.options = previous
+      end
+
+      def _dep_options_for(command)
+        Dispatch.dep_options(self.class.class_options.merge(command.options), _typed_declared, _raw_options, options)
+      end
+
+      # One failure: a dependency that exited ends asgard with its status,
+      # as it would have on its own; anything else propagates unchanged.
+      def _raise_dep_failure(error)
+        error = error.error while error.is_a?(DepFailed) && error.error
+        case error
+        when DepExit, SystemExit then exit(error.status)
+        else raise(error)
+        end
+      end
+
+      # Every script in the group stopped because asgard was signalled:
+      # that is the one failure, so it exits as asgard would on its own.
+      def _raise_group_failure(errors)
+        signalled = Shell::Children.exit_status
+        exit(signalled) if signalled
+        errors.each { |task, e| $stderr.puts "asgard: dependency '#{task}' failed: #{_describe_failure(e)}" } # rubocop:disable Style/StderrPuts
+        raise Asgard::Error, "parallel dependencies failed: #{errors.keys.join(', ')}"
+      end
+
+      def _describe_failure(error) = error.is_a?(DepExit) ? error.message : "#{error.class}: #{error.message}"
+
+      def _signal_done(target, result, error = nil)
         klass = self.class
         klass._ran_mutex.synchronize do
           klass._done.add(target)
           klass._results[target] = result
+          klass._failed[target]  = error if error
           klass._cond[target].broadcast
         end
       end
 
-      def run_dep(task)
+      def _run_dep(task)
         command = self.class.all_commands[task.to_s]
-        invoke_command(command) if command
+        _with_options(_dep_options_for(command)) { invoke_command(command) } if command
       end
     end
   end

data/lib/asgard/base/registry.rb

@@ -7,13 +7,16 @@ module Asgard
     # --doctor`) uses to detect a silently-overridden `def`, same file or
     # across files.
     module Registry
-      def subclasses
-        @subclasses ||= []
+      # Every subclass of Asgard::Base, at any depth, in definition order.
+      # Not `subclasses`: that is Ruby's own Class#subclasses (direct
+      # subclasses only), which a .loki may well call.
+      def _asgard_subclasses
+        @_asgard_subclasses ||= []
       end
 
       def inherited(subclass)
         super
-        Asgard::Base.subclasses << subclass
+        Asgard::Base._asgard_subclasses << subclass
         subclass.instance_variable_set(:@_deps,              {})
         subclass.instance_variable_set(:@_method_log,        Hash.new { |h, k| h[k] = [] })
         subclass.instance_variable_set(:@_pending_deps,      [])
@@ -23,6 +26,7 @@ module Asgard
         subclass.instance_variable_set(:@_done,       Set.new)
         subclass.instance_variable_set(:@_cond,       Hash.new { |h, k| h[k] = ConditionVariable.new })
         subclass.instance_variable_set(:@_ran_mutex,  Mutex.new)
+        subclass.instance_variable_set(:@_failed,     {})
       end
 
       # Every source_location a method name has ever been defined at, in

data/lib/asgard/base.rb

@@ -17,11 +17,45 @@ module Asgard
     extend TaskDSL
     include Dispatch
 
+    THOR_LIB   = File.dirname(Thor.method(:subcommand).source_location.first)
+    ASGARD_LIB = File.expand_path("..", __dir__)
+
+    # Names Thor's machinery relies on: its reserved words plus the readers
+    # dispatch calls on every instance, which Thor itself does not guard.
+    # A task or helper by one of these names breaks every command, with
+    # nothing in the failure pointing at the cause.
+    RESERVED_NAMES = (Thor::THOR_RESERVED_WORDS + %w[args parent_options]).map(&:to_sym).freeze
+
+    # The helpers lib/asgard/kernel_methods.rb adds to Kernel.
+    KERNEL_HELPERS = %i[debug? verbose? env changes loki_up import import_up].freeze
+
+    # Asgard's own helpers a task body calls (sh, env, dep_result, ...)
+    # and the methods dispatch runs every task through. A task or helper
+    # by one of these names would silently replace it.
+    ASGARD_HELPERS = [Shell, Dispatch].flat_map { |mod| mod.instance_methods(false) + mod.private_instance_methods(false) }
+                                      .concat(KERNEL_HELPERS).uniq.freeze
+
     class << self
       # A usage error (missing argument, unknown command or option) exits 1,
       # so CI and `asgard do` sequences see the failure. Thor's default is 0.
       def exit_on_failure? = true
 
+      # A subcommand group's help lists only the group's own commands. The
+      # tasks it inherits from Tasks still run, but Thor would list them
+      # too, under the class's default namespace (doc_commands for
+      # DocCommands), burying the group's own.
+      def printable_commands(all = true, subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
+        super(all && !subcommand, subcommand)
+      end
+
+      # `subcommand "doc", DocCommands` names the group doc, so `tree` shows
+      # it as doc rather than Thor's doc_commands. A group that set its own
+      # namespace keeps it.
+      def subcommand(name, klass)
+        klass.namespace(name) if klass.namespace == Thor::Util.namespace_from_thor_class(klass)
+        super
+      end
+
       def header(text = nil)
         return @_header if text.nil?
         (@_header ||= []) << text
@@ -32,22 +66,119 @@ module Asgard
         (@_footer ||= []).unshift(text)
       end
 
+      # A method Thor generates (a `subcommand` dispatcher, a subcommand
+      # class's string-eval'd `help`) reports a source_location inside Thor.
+      # Credit it to the first caller frame outside Thor and asgard — the
+      # .loki line that called `subcommand` — so Doctor lists it there.
+      def definition_site(loc, callers)
+        return loc unless thor_generated?(loc.first)
+
+        site = callers.find { |frame| user_frame?(frame.path) }
+        site ? [site.path, site.lineno] : loc
+      end
+
+      def thor_generated?(path)
+        path.start_with?("(eval", THOR_LIB)
+      end
+
+      def user_frame?(path)
+        !thor_generated?(path) && !path.start_with?(ASGARD_LIB)
+      end
+
+      HIDDEN_DESC = "(hidden task: defined without a desc)"
+
+      # A public method a .loki defines without a desc is a hidden task:
+      # it runs from the command line and as a dependency, and help leaves
+      # it out. Not one: a method that already has a desc pending, anything
+      # inside no_commands (helpers), non-public methods, the gem's own
+      # _-prefixed names, and a name Thor already has a command for (help).
+      # Nor a method every object already has (to_s, inspect,
+      # respond_to_missing?): see object_method?.
+      def implicit_hidden_task?(method_name)
+        name = method_name.to_s
+        !@usage && !no_commands? && !name.start_with?("_") && !object_method?(method_name) &&
+          public_method_defined?(method_name) && !all_commands.key?(name)
+      end
+
+      # A method Thor (and so Object) already defines: a public one
+      # (to_s, inspect, display), or a private one that is not a Kernel
+      # module function (method_missing, respond_to_missing?). Kernel's
+      # module functions (test, format, system) stay free to be task names.
+      def object_method?(method_name)
+        Thor.public_method_defined?(method_name) ||
+          (Thor.private_method_defined?(method_name) && !Kernel.respond_to?(method_name))
+      end
+
+      # The task `asgard help NAME` describes: a built-in group's mapped
+      # name ("jobs") is its registered subcommand ("_jobs"); anything else
+      # is NAME unchanged.
+      def help_target(command)
+        mapped = map[command.to_s]&.to_s
+        subcommands.include?(mapped) ? mapped : command
+      end
+
+      def _check_reserved_name!(method_name)
+        thor = RESERVED_NAMES.include?(method_name)
+        return unless thor || ASGARD_HELPERS.include?(method_name)
+
+        file, line = instance_method(method_name).source_location
+        remove_method(method_name) # leave the original in place, so the class still dispatches
+        what = thor ? "a name Thor reserves" : "an asgard helper"
+        raise Asgard::Error, "'#{method_name}' at #{file}:#{line} is #{what} and cannot be a task or helper; rename it"
+      end
+
+      # `private def helper = ...` makes the method private only after
+      # method_added has seen it public and made it a hidden task, and Ruby
+      # has no hook for the change (overriding `private` would break its
+      # no-argument form in every subclass). So whenever the command table
+      # is read (dispatch, help, doctor), a hidden task whose method is no
+      # longer public is dropped from it.
+      def all_commands
+        super.tap { _drop_non_public_hidden_tasks(it) }
+      end
+
+      # Removes from +all+ (and this class's own table) each hidden task,
+      # one defined without a desc, whose method is private or protected,
+      # with any depends_on attached to it.
+      def _drop_non_public_hidden_tasks(all)
+        stale = all.select { |name, command| command.description == HIDDEN_DESC && !public_method_defined?(name) }
+        stale.each_key do |name|
+          all.delete(name)
+          commands.delete(name)
+          _deps.delete(name.to_sym)
+        end
+      end
+
+      # A redefined to_s or inspect with no desc of its own: not a task, and
+      # not worth Thor's "without usage or description" warning either.
+      def plain_object_method?(method_name) = !@usage && !@_pending_single_desc && object_method?(method_name)
+
+      # Turns a pending single-argument `desc "text"` into Thor's desc for
+      # the method just defined.
+      def _apply_pending_desc(name)
+        return unless @_pending_single_desc && !no_commands?
+
+        pending_desc = @_pending_single_desc
+        pending_opts = @_pending_single_desc_opts || {}
+        @_pending_single_desc      = nil
+        @_pending_single_desc_opts = nil
+        desc(name, pending_desc, pending_opts)
+      end
+
       def method_added(method_name)
+        _check_reserved_name!(method_name)
+        return no_commands { super } if plain_object_method?(method_name)
+
         name           = method_name.to_s
         private_method = name.start_with?("_")
 
         unless private_method
           loc = instance_method(method_name).source_location
-          _method_log[method_name] << loc if loc
+          _method_log[method_name] << definition_site(loc, caller_locations) if loc
         end
 
-        if @_pending_single_desc && !no_commands?
-          pending_desc = @_pending_single_desc
-          pending_opts = @_pending_single_desc_opts || {}
-          @_pending_single_desc      = nil
-          @_pending_single_desc_opts = nil
-          desc(name, pending_desc, pending_opts)
-        end
+        _apply_pending_desc(name)
+        desc(name, HIDDEN_DESC, hide: true) if implicit_hidden_task?(method_name)
 
         return super unless @usage
 
@@ -70,7 +201,7 @@ module Asgard
 
       say header_text.join("\n\n") if header_text && top_level
       say "\n"
-      super
+      super(klass.help_target(command), subcommand)
       say footer_text.join("\n\n") if footer_text && top_level
     end
 

data/lib/asgard/declarations.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Asgard
+  # What `schedule` and `watch` share: a registry of declared entries,
+  # keyed by entry name.
+  module Declarations
+    module_function
+
+    # Stores +spec+ (a Hash with a :name) in +declarations+ and returns it.
+    # An identical entry already there is a no-op (a .loki file loaded
+    # twice); a different one under the same name is an error that names
+    # +kind+ ("schedule", "watch").
+    def record(declarations, spec, kind)
+      name  = spec[:name]
+      taken = declarations[name]
+      raise ArgumentError, "#{kind}: #{name.inspect} is already declared; give one a distinct as: name" if taken && taken != spec
+
+      declarations[name] = spec
+    end
+  end
+end

data/lib/asgard/doctor/task_sections.rb

@@ -7,6 +7,14 @@ module Asgard
     # more than one recorded location (the each/each_gem class of bug) —
     # earlier ones OVERRIDDEN (dead), the last one the active redefinition.
     module TaskSections
+      # A hidden task (one defined without a desc) says so, after any
+      # override note.
+      def self.with_hidden(status, hidden)
+        return status unless hidden
+
+        [status, "hidden (no desc; not shown by help)"].compact.join("; ")
+      end
+
       private
 
       def build_task_sections
@@ -20,12 +28,12 @@ module Asgard
         file_map = Hash.new { |h, k| h[k] = [] }
         @loaded_files.each { |f| file_map[f] }
 
-        commands = klass.all_commands.keys
+        commands = klass.all_commands
         klass._method_log.each do |name, locations|
-          next unless commands.include?(name.to_s)
+          command = commands[name.to_s] or next
 
           locations.each_with_index do |(file, line), idx|
-            file_map[file] << [name, line, task_status(locations, idx)]
+            file_map[file] << [name, line, TaskSections.with_hidden(task_status(locations, idx), command.hidden?)]
           end
         end
         file_map.each_value { |tasks| tasks.sort_by! { |t| t[1] } }