superkick 0.1.0

data/lib/superkick/supervisor.rb

@@ -0,0 +1,516 @@
+# frozen_string_literal: true
+
+module Superkick
+  # Supervises all poller lifecycles — monitor threads (per-agent) and spawner
+  # threads (server-level) — in response to commands from Control::Server.
+  #
+  # Commands processed from the queue:
+  #   :register        → start all monitors for a newly registered agent
+  #   :add_monitor     → start one monitor for an existing agent
+  #   :remove_monitor  → stop one monitor
+  #   :restart_monitors → stop then restart all monitors for an agent
+  #   :start_spawner   → start a spawner via IPC
+  #
+  # At server startup, call restore_from_registry to resume monitors for all
+  # agents already persisted (e.g. after a server restart).
+  class Supervisor
+    attr_reader :agent_threads, :spawner_threads, :worker, :goal_threads, :cost_poller_threads
+
+    def initialize(store:, injector:, notification_dispatcher:, buffer_client: nil, config: Superkick.config, agent_spawner: nil, approval_store: nil, team_log_store: nil)
+      @store = store
+      @injector = injector
+      @config = config
+      @approval_store = approval_store
+      @notification_dispatcher = notification_dispatcher
+      @team_log_store = team_log_store
+      @buffer_client = buffer_client || Buffer.client_from(store:, config:)
+      self.agent_spawner = agent_spawner
+      @command_queue = Queue.new
+      @worker = nil
+      @agent_threads = {}
+      @agent_threads_mutex = Mutex.new
+      @spawner_threads = {}
+      @spawners = {}
+      @goal_threads = {}
+      @goals = {}
+      @goals_mutex = Mutex.new
+      @goal_paused = {}
+      @goal_elapsed = {}
+      @goal_started_at = {}
+      @cost_poller_threads = {}
+    end
+
+    def start
+      @worker = Thread.new { work_loop }
+      self
+    end
+
+    def agent_spawner=(spawner)
+      @agent_spawner = spawner
+      @workflow_executor = spawner ? Spawn::WorkflowExecutor.new(
+        store: @store, agent_spawner: spawner, notification_dispatcher: @notification_dispatcher,
+        config: @config
+      ) : nil
+    end
+
+    def stop
+      @command_queue << {action: :shutdown}
+      @worker&.join(5)
+
+      # Stop all agent monitor threads
+      @agent_threads_mutex.synchronize do
+        @agent_threads.each_value { it.each_value(&:kill) }
+        @agent_threads.clear
+      end
+
+      # Stop all goal checker threads
+      @goals_mutex.synchronize do
+        @goal_threads.each_value(&:kill)
+        @goal_threads.clear
+        @goals.each_value(&:teardown)
+        @goals.clear
+      end
+
+      # Stop all cost poller threads
+      @cost_poller_threads.each_value(&:kill)
+      @cost_poller_threads.clear
+
+      # Stop all spawner threads
+      @spawner_threads.each_value(&:kill)
+      @spawner_threads.clear
+      @spawners.clear
+    end
+
+    # Control::Server calls this to enqueue commands.
+    def enqueue(action, **params)
+      @command_queue << {action:}.merge(params)
+    end
+
+    # Call once at server boot to re-create monitor threads for persisted agents.
+    def restore_from_registry
+      @store.each do |agent|
+        agent.monitors.each_key { start_agent_monitor(agent.id, it) }
+      end
+    end
+
+    # Return status info for all spawners.
+    def spawner_status
+      @spawners.map do |name, spawner|
+        thread = @spawner_threads[name]
+        config = @config.spawners[name] || {}
+        info = {
+          name:,
+          type: spawner.class.type.to_s,
+          status: thread&.alive? ? "running" : "stopped"
+        }
+        info[:on_complete_target] = workflow_target_name(config[:on_complete]) if config[:on_complete]
+        info[:on_fail_target] = workflow_target_name(config[:on_fail]) if config[:on_fail]
+        info
+      end
+    end
+
+    # Stop a single spawner by name.
+    def stop_spawner(name)
+      thread = @spawner_threads.delete(name)
+      thread&.kill
+      @spawners.delete(name)
+      Superkick.logger.info("supervisor") { "Stopped spawner #{name}" }
+    end
+
+    # Start spawner threads for each configured spawner.
+    def start_spawners
+      @config.spawners.each do |name, config|
+        start_spawner_monitor(name, config)
+      end
+    end
+
+    # Returns the goal instance for a given agent, or nil.
+    def goal_for(agent_id)
+      @goals_mutex.synchronize { @goals[agent_id] }
+    end
+
+    # Called by Control::Server when a superkick_signal_goal arrives for an agent.
+    # Forwards the signal to the AgentSignal goal if one exists.
+    def signal_goal(agent_id, status)
+      @goals_mutex.synchronize do
+        goal = @goals[agent_id]
+        goal.signal!(status) if goal.respond_to?(:signal!)
+      end
+    end
+
+    # Pause the goal checker for a claimed agent. The thread continues to
+    # sleep but skips check logic. Elapsed time is snapshotted so
+    # max_duration doesn't tick during the pause.
+    def pause_goal_checker(agent_id)
+      @goals_mutex.synchronize do
+        @goal_paused[agent_id] = true
+        if @goal_started_at[agent_id]
+          @goal_elapsed[agent_id] = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @goal_started_at[agent_id]
+        end
+      end
+      Superkick.logger.info("supervisor") { "Goal checker paused for #{agent_id}" }
+    end
+
+    # Resume the goal checker for an unclaimed agent. Restores started_at
+    # so the remaining max_duration budget is preserved.
+    def resume_goal_checker(agent_id)
+      @goals_mutex.synchronize do
+        @goal_paused.delete(agent_id)
+        if @goal_elapsed[agent_id]
+          @goal_started_at[agent_id] = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @goal_elapsed[agent_id]
+          @goal_elapsed.delete(agent_id)
+        end
+      end
+      Superkick.logger.info("supervisor") { "Goal checker resumed for #{agent_id}" }
+    end
+
+    # Returns whether the goal checker for an agent is paused.
+    def goal_paused?(agent_id)
+      @goals_mutex.synchronize { @goal_paused[agent_id] == true }
+    end
+
+    # Start a goal checker for a spawned agent.
+    def start_goal_checker(agent_id:, goal_config:, spawner_config:)
+      @goals_mutex.synchronize do
+        return if @goal_threads[agent_id]&.alive?
+      end
+
+      goal = Goal.build(goal_config, agent_id:)
+      @goals_mutex.synchronize { @goals[agent_id] = goal }
+
+      interval = goal_config[:check_interval] || @config.poll_interval
+      max_duration = spawner_config[:max_duration]
+
+      stall_threshold = spawner_config[:stall_threshold]
+
+      @goals_mutex.synchronize do
+        @goal_threads[agent_id] = Thread.new do
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          @goals_mutex.synchronize { @goal_started_at[agent_id] = started_at }
+          stalled_notified = false
+          Superkick.logger.info("supervisor") { "Goal checker started for #{agent_id} (type=#{goal_config[:type]})" }
+
+          loop do
+            sleep(interval)
+
+            # Skip checks when paused (claimed agent)
+            if @goals_mutex.synchronize { @goal_paused[agent_id] }
+              next
+            end
+
+            # Check max_duration timeout using tracked started_at
+            if max_duration
+              current_started_at = @goals_mutex.synchronize { @goal_started_at[agent_id] }
+              elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - current_started_at
+              if elapsed >= max_duration
+                Superkick.logger.warn("supervisor") { "Agent #{agent_id} exceeded max_duration (#{max_duration}s) — terminating" }
+                agent = @store.get(agent_id)
+                agent&.set_goal_status(:timed_out)
+                notify_lifecycle(:agent_timed_out, agent_id:,
+                  spawner_name: spawner_config[:name],
+                  message: "Agent #{agent_id} timed out after #{max_duration}s")
+
+                # If this is a team lead, terminate all team members
+                if agent&.team_id && agent&.team_role == :lead
+                  terminate_team(agent.team_id, reason: :timed_out)
+                else
+                  @agent_spawner&.terminate(agent_id)
+                end
+                break
+              end
+            end
+
+            # Check goal status
+            status = goal.check
+            next if status == :pending
+
+            agent = @store.get(agent_id)
+            agent&.set_goal_status(status)
+
+            if Goal::TERMINAL_STATUSES.include?(status)
+              Superkick.logger.info("supervisor") { "Agent #{agent_id} goal terminal: #{status}" }
+              notify_lifecycle_for_goal(status, agent_id:, spawner_name: spawner_config[:name])
+              @workflow_executor&.fire(agent_id:, goal_status: status, spawner_config:)
+
+              # If this is a team lead, terminate all team members
+              agent = @store.get(agent_id)
+              if agent&.team_id && agent&.team_role == :lead
+                terminate_team(agent.team_id, reason: status)
+              else
+                @agent_spawner&.terminate(agent_id)
+              end
+              break
+            elsif status == :errored
+              Superkick.logger.info("supervisor") { "Agent #{agent_id} goal status: #{status}" }
+              notify_lifecycle(:agent_blocked, agent_id:,
+                spawner_name: spawner_config[:name],
+                message: "Agent #{agent_id} needs help (goal status: errored)")
+            else
+              Superkick.logger.info("supervisor") { "Agent #{agent_id} goal status: #{status}" }
+            end
+
+            # Stall detection
+            if stall_threshold
+              state = query_idle_state(agent_id)
+              if state && state[:seconds_idle] && state[:seconds_idle] >= stall_threshold
+                unless stalled_notified
+                  notify_lifecycle(:agent_stalled, agent_id:,
+                    spawner_name: spawner_config[:name],
+                    message: "Agent #{agent_id} idle for #{state[:seconds_idle].round}s")
+                  stalled_notified = true
+                end
+              else
+                stalled_notified = false
+              end
+            end
+          rescue => e
+            Superkick.logger.error("supervisor") { "Goal check error for #{agent_id}: #{e.message}" }
+          end
+        ensure
+          @goals_mutex.synchronize do
+            @goals.delete(agent_id)&.teardown
+            @goal_threads.delete(agent_id)
+            @goal_paused.delete(agent_id)
+            @goal_elapsed.delete(agent_id)
+            @goal_started_at.delete(agent_id)
+          end
+        end
+      end
+    end
+
+    # Stop a goal checker for a given agent.
+    def stop_goal_checker(agent_id)
+      thread = @goals_mutex.synchronize { @goal_threads.delete(agent_id) }
+      thread&.kill
+      @goals_mutex.synchronize { @goals.delete(agent_id)&.teardown }
+      stop_cost_poller(agent_id)
+    end
+
+    # Start a cost poller for a spawned agent.
+    def start_cost_poller(agent_id:, spawner_config: nil)
+      return if @cost_poller_threads[agent_id]&.alive?
+
+      poller = CostPoller.new(agent_id:, store: @store)
+      @cost_poller_threads[agent_id] = Thread.new do
+        Superkick.logger.info("supervisor") { "Cost poller started for #{agent_id}" }
+        poller.run
+      rescue => e
+        Superkick.logger.error("supervisor") { "Cost poller error (#{agent_id}): #{e.message}" }
+      end
+    end
+
+    # Stop a cost poller for a given agent.
+    def stop_cost_poller(agent_id)
+      thread = @cost_poller_threads.delete(agent_id)
+      thread&.kill
+    end
+
+    # Terminate all agents in a team. Called when the team lead times out
+    # or the team goal reaches a terminal status.
+    def terminate_team(team_id, reason: :timed_out)
+      teammates = @store.team(team_id)
+      Superkick.logger.info("supervisor") { "Terminating team #{team_id} (#{teammates.size} agents, reason: #{reason})" }
+
+      teammates.each do |agent|
+        agent.set_goal_status(reason)
+        @agent_spawner&.terminate(agent.id)
+      end
+
+      event_type = case reason
+      when :completed then :team_completed
+      when :failed then :team_failed
+      when :timed_out then :team_timed_out
+      else :team_failed
+      end
+
+      event = {
+        event_type:,
+        monitor_type: :spawner,
+        monitor_name: :system,
+        team_members: teammates.map(&:id)
+      }
+      dispatch_notification(
+        event:,
+        agent_id: team_id,
+        message: "Team #{team_id} #{reason} (#{teammates.size} agents)"
+      )
+    end
+
+    private
+
+    # Extract a human-readable workflow target name from a workflow config.
+    def workflow_target_name(workflow_config)
+      workflow_config&.dig(:spawner)&.to_s || "inline"
+    end
+
+    # Dispatch a lifecycle notification event.
+    # The Team::LogNotifier handles writing to the team log.
+    def notify_lifecycle(event_type, agent_id:, spawner_name: nil, message: nil)
+      event = {
+        event_type:,
+        monitor_type: :spawner,
+        monitor_name: spawner_name || :system
+      }
+      msg = message || "Superkick: #{event_type} for #{agent_id}"
+      dispatch_notification(event:, agent_id:, message: msg)
+    end
+
+    # Map a terminal goal status to the appropriate lifecycle event.
+    def notify_lifecycle_for_goal(status, agent_id:, spawner_name: nil)
+      event_type = case status
+      when :completed then :agent_completed
+      when :failed then :agent_failed
+      when :timed_out then :agent_timed_out
+      else return # non-terminal statuses don't get lifecycle notifications here
+      end
+      notify_lifecycle(event_type, agent_id:, spawner_name:,
+        message: "Agent #{agent_id} goal #{status}")
+    end
+
+    # Query idle state from the agent via the buffer client.
+    def query_idle_state(agent_id)
+      return nil unless @buffer_client.reachable?(agent_id)
+
+      response = @buffer_client.request(agent_id, "idle_state")
+      response&.fetch(:ok, false) ? response : nil
+    rescue Buffer::Client::AgentUnreachable, SystemCallError, IOError, JSON::ParserError => e
+      Superkick.logger.debug("supervisor:#{agent_id}") { "Idle state query error: #{e.message}" }
+      nil
+    end
+
+    def work_loop
+      loop do
+        command = @command_queue.pop
+        process(command)
+        break if command[:action] == :shutdown
+      rescue => e
+        Superkick.logger.error("supervisor") { "Supervisor error: #{e.message}\n#{e.backtrace.first(5).join("\n")}" }
+      end
+    end
+
+    def process(command)
+      agent_id = command[:agent_id]
+      name = command[:monitor_name]
+
+      case command[:action]
+      when :register then start_all_agent_monitors(agent_id)
+      when :add_monitor then start_agent_monitor(agent_id, name)
+      when :remove_monitor then stop_agent_monitor(agent_id, name)
+      when :restart_monitors then restart_agent_monitors(agent_id)
+      when :unregister
+        stop_all_agent_monitors(agent_id)
+        stop_goal_checker(agent_id)
+      when :start_spawner
+        config = command[:config] || @config.spawners[name]
+        start_spawner_monitor(name, config) if config
+      when :shutdown then nil
+      end
+    end
+
+    def start_agent_monitor(agent_id, name)
+      name_sym = name.to_sym
+      @agent_threads_mutex.synchronize do
+        t = @agent_threads.dig(agent_id, name_sym)
+        return if t&.alive?
+      end
+
+      agent = @store.get(agent_id)
+      return unless agent
+
+      config = agent.monitor_config(name) || {}
+      type_key = config[:type]&.to_sym || name_sym
+
+      klass = Monitor.registered[type_key]
+      unless klass
+        Superkick.logger.warn("supervisor") { "Unknown monitor type '#{type_key}' — skipping" }
+        return
+      end
+
+      # Resolve missing config values from the agent's environment snapshot.
+      config = klass.resolve_config(config, environment: agent.environment) if agent.environment
+
+      handler = InjectHandler.new(injector: @injector, agent:)
+      server_context = {team_log_store: @team_log_store}
+      monitor = klass.new(name: name_sym, agent:, config:, handler:, server_context:)
+
+      thread = Thread.new do
+        Superkick.logger.info("supervisor") { "Starting #{name_sym} monitor (type=#{type_key}) for #{agent_id}" }
+        monitor.run
+      rescue => e
+        Superkick.logger.error("supervisor") { "Monitor thread error (#{name_sym}/#{agent_id}): #{e.message}\n#{e.backtrace.first(5).join("\n")}" }
+      ensure
+        @agent_threads_mutex.synchronize do
+          @agent_threads[agent_id]&.delete(name_sym)
+          threads = @agent_threads[agent_id]
+          @agent_threads.delete(agent_id) if threads && threads.empty?
+        end
+      end
+
+      @agent_threads_mutex.synchronize do
+        (@agent_threads[agent_id] ||= {})[name_sym] = thread
+      end
+    end
+
+    def stop_agent_monitor(agent_id, name)
+      name_sym = name.to_sym
+      thread = @agent_threads_mutex.synchronize { @agent_threads.dig(agent_id, name_sym) }
+      return unless thread
+
+      @agent_threads_mutex.synchronize { @agent_threads[agent_id]&.delete(name_sym) }
+      thread.kill
+      Superkick.logger.info("supervisor") { "Stopped #{name_sym} monitor for #{agent_id}" }
+    end
+
+    def start_all_agent_monitors(agent_id)
+      agent = @store.get(agent_id)
+      return unless agent
+
+      agent.monitors.each_key { start_agent_monitor(agent_id, it) }
+    end
+
+    def stop_all_agent_monitors(agent_id)
+      threads = @agent_threads_mutex.synchronize { @agent_threads.delete(agent_id) }
+      return unless threads
+
+      threads.each_value(&:kill)
+    end
+
+    def restart_agent_monitors(agent_id)
+      stop_all_agent_monitors(agent_id)
+      start_all_agent_monitors(agent_id)
+    end
+
+    def start_spawner_monitor(name, config)
+      return if @spawner_threads[name]&.alive?
+
+      type_key = config[:type]&.to_sym || name.to_sym
+      klass = Spawner.registered[type_key]
+      unless klass
+        Superkick.logger.warn("supervisor") { "Unknown spawner type '#{type_key}' — skipping" }
+        return
+      end
+
+      handler = Spawn::Handler.new(
+        spawner: @agent_spawner,
+        store: @store,
+        spawner_config: config,
+        approval_store: @approval_store,
+        notification_dispatcher: @notification_dispatcher,
+        repository_source: Superkick.config.repository_source
+      )
+      spawner = klass.new(name:, config:, handler:)
+      @spawners[name] = spawner
+
+      @spawner_threads[name] = Thread.new do
+        Superkick.logger.info("supervisor") { "Starting spawner #{name} (type=#{type_key})" }
+        spawner.run
+      rescue => e
+        Superkick.logger.error("supervisor") { "Spawner thread error (#{name}): #{e.message}" }
+      end
+    end
+
+    def dispatch_notification(...)
+      @notification_dispatcher.dispatch(...)
+    end
+  end
+end

data/lib/superkick/team/artifact_store.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Superkick
+  module Team
+    # ArtifactStore — per-team artifact storage for sharing data between agents.
+    #
+    # Artifacts are persisted to disk at ~/.superkick/teams/<team_id>/artifacts/<author>--<name>.json.
+    # Each artifact is a JSON file with name, author, content, and timestamps.
+    # Thread-safe via Mutex.
+    class ArtifactStore
+      Artifact = Data.define(:name, :author, :content, :created_at, :updated_at)
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      # Publish (create or update) an artifact.
+      def publish(team_id:, author:, name:, content:)
+        @mutex.synchronize do
+          path = artifact_path(team_id, author, name)
+          FileUtils.mkdir_p(File.dirname(path))
+
+          now = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ")
+          existing = load_artifact(path)
+          created_at = existing ? existing[:created_at] : now
+
+          data = {name:, author:, content:, created_at:, updated_at: now}
+          File.write(path, JSON.pretty_generate(data))
+
+          Artifact.new(**data)
+        end
+      end
+
+      # Read an artifact by author and name.
+      def get(team_id:, author:, name:)
+        @mutex.synchronize do
+          data = load_artifact(artifact_path(team_id, author, name))
+          data ? Artifact.new(**data) : nil
+        end
+      end
+
+      # List artifacts for a team, optionally filtered by author.
+      # Returns metadata only (no content).
+      def list(team_id:, author: nil)
+        @mutex.synchronize do
+          dir = artifacts_dir(team_id)
+          return [] unless Dir.exist?(dir)
+
+          pattern = author ? "#{sanitize(author)}--*.json" : "*.json"
+          Dir.glob(File.join(dir, pattern)).filter_map do |path|
+            data = load_artifact(path)
+            next unless data
+            {name: data[:name], author: data[:author], created_at: data[:created_at], updated_at: data[:updated_at]}
+          end
+        end
+      end
+
+      # Delete an artifact.
+      def delete(team_id:, author:, name:)
+        @mutex.synchronize do
+          path = artifact_path(team_id, author, name)
+          return false unless File.exist?(path)
+          FileUtils.rm_f(path)
+          true
+        end
+      end
+
+      private
+
+      def artifacts_dir(team_id)
+        File.join(Superkick.config.teams_dir, team_id.to_s, "artifacts")
+      end
+
+      def artifact_path(team_id, author, name)
+        File.join(artifacts_dir(team_id), "#{sanitize(author)}--#{sanitize(name)}.json")
+      end
+
+      def sanitize(str)
+        str.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
+      end
+
+      def load_artifact(path)
+        return nil unless File.exist?(path)
+        JSON.parse(File.read(path), symbolize_names: true)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+
+  # Backwards-compatible alias
+end