samagotchi 0.58.0 → 0.59.0

data/lib/samagotchi/tools/execute.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
-require "open3"
+require "fileutils"
+require_relative "../config"
+require_relative "../paths"
 require_relative "../process_group"
 require_relative "output_guardrails"
 require_relative "shell"
+require_relative "task_runtime"
 
 module Samagotchi
   module Tools
@@ -24,8 +27,11 @@ module Samagotchi
       NOT_RUN_ON_STOP = "Error: not run, the user stopped the turn"
       # The first words of a command the user's Stop killed.
       STOPPED_BY_USER = "Error: command stopped by the user"
-      # The first words of a command killed at the limit (hooks match on it).
+      # The first words of a command killed at the limit in kill mode (hooks
+      # match on it).
       TIMEOUT_PREFIX = "Error: command timed out after"
+      # Only in kill mode (execute.on_timeout: kill, the !cmd bang): in task
+      # mode a command at the limit is handed over (HANDED_OVER).
       TIMEOUT_HINT = "(killed at the limit; for a long command use task_create, then task_wait)"
       # How long the output may stay open once the shell has exited: a
       # `server &` child keeps the pipes, and the read would never end.
@@ -35,23 +41,52 @@ module Samagotchi
       # It had left the process group (setsid), so it is still running.
       BACKGROUND_DETACHED_HINT = "(a background process kept the output open and was left running; " \
                                  "to keep a server or long job running, start it with task_create)"
-      READ_CHUNK_BYTES = 65_536
+      # What a command still running at the limit gets (execute.on_timeout):
+      # :task hands it over as a task (HANDED_OVER), :kill stops it.
+      ON_TIMEOUT = %i[task kill].freeze
+      # How many of the last lines of each log a hand-over shows.
+      HAND_OVER_TAIL_LINES = 10
+      # Where each command's logs and exit_code file live while it runs, and
+      # after, for a handed-over one: <state dir>/execute/<id>/ (outside the
+      # workspace, so git and `rm -rf tmp` never see them).
+      LOGS_DIRNAME = "execute"
+      STDOUT_LOG = "stdout.log"
+      STDERR_LOG = "stderr.log"
+      # A handed-over command's dir names its task.json here (the sweep).
+      BACK_POINTER = "task"
+      # The sweep (sweep_leftovers): a dir no command holds any more, with
+      # no back-pointer after a day (chi died mid-execute), or whose task
+      # record is gone or is older than RETENTION_DAYS.
+      ORPHAN_AGE_SEC = 24 * 3600
+      RETENTION_DAYS = 14
+      # The first words of a command still running at the limit and handed
+      # over as a task (handed_over_line). TaskRuntime.handed_over_id, the
+      # web's task_stop.js and the slow-commands bundle match on them.
+      HANDED_OVER = "Still running at the"
 
       def self.name        = NAME
 
+      # A hand-over result's first line; its "task_id: …" line follows it.
+      def self.handed_over_line(limit_sec)
+        "#{HANDED_OVER} #{Integer(limit_sec)}s limit: moved to a background task, not killed."
+      end
+
       # @param cancelled [#call] true once the user stopped the turn: the
       #   command is killed (or not started) and its output so far returned
       # @param env [Hash, nil] added to the command's environment
-      def self.call(command, cwd: nil, cancelled: -> { false }, env: nil)
+      # @param on_timeout [Symbol, String, nil] :task or :kill (ON_TIMEOUT);
+      #   nil reads execute.on_timeout
+      def self.call(command, cwd: nil, cancelled: -> { false }, env: nil, on_timeout: nil)
         return NOT_RUN_ON_STOP if cancelled.call
 
         command = command.strip
         resolved_cwd = resolve_cwd(cwd)
         return "Error: cwd not found: #{resolved_cwd}" unless resolved_cwd
 
+        sweep_leftovers_once
         timeout_sec = timeout_seconds
         stdout, stderr, status, held_open = run_command(command, timeout_sec: timeout_sec, cwd: resolved_cwd, cancelled: cancelled,
-                                                      env: env)
+                                                                 env: env, on_timeout: on_timeout_mode(on_timeout))
 
         stdout_block = output_block("stdout", stdout)
         stderr_block = output_block("stderr", stderr)
@@ -67,6 +102,8 @@ module Samagotchi
         parts << (silent ? "exit: #{status.exitstatus} (no output)" : "exit: #{status.exitstatus}")
         parts << (held_open == :detached ? BACKGROUND_DETACHED_HINT : BACKGROUND_HINT) if held_open
         parts.join("\n")
+      rescue HandedOver => e
+        e.text
       rescue CommandTimedOut => e
         # Keep the "Error:" first line (callers classify on it), say how to run
         # a long command, and return whatever it printed before it was killed.
@@ -77,6 +114,14 @@ module Samagotchi
         "Error: #{e.message}"
       end
 
+      # execute.on_timeout, or the caller's: :task (the default) or :kill.
+      def self.on_timeout_mode(value)
+        value = Samagotchi::Config.get("execute.on_timeout") if value.nil? || value.to_s.strip.empty?
+        mode = value.to_s.strip.downcase.to_sym
+        ON_TIMEOUT.include?(mode) ? mode : :task
+      end
+      private_class_method :on_timeout_mode
+
       def self.stopped_result(first_line, error)
         parts = [first_line]
         parts << output_block("stdout", error.stdout)
@@ -85,58 +130,91 @@ module Samagotchi
       end
       private_class_method :stopped_result
 
-      def self.run_command(command, timeout_sec:, cwd:, cancelled: -> { false }, env: nil)
-        stdout_text = ""
-        stderr_text = ""
+      # The command's stdout and stderr go to two logs in its dir under the
+      # state dir (logs_root), not to pipes: a command handed over at the
+      # limit keeps writing there after execute (or chi) is gone. chi holds
+      # each log with a shared flock across the spawn, so the command's
+      # processes inherit open files a fresh exclusive flock can't get while
+      # any of them still has one (held?). stdin is /dev/null: a background
+      # group reading the terminal would stop on SIGTTIN. The shell is
+      # detached at once; the waiter it returns reaps it, also after a
+      # hand-over, so it never stays a zombie.
+      def self.run_command(command, timeout_sec:, cwd:, cancelled: -> { false }, env: nil, on_timeout: :task)
         status = nil
         timed_out = false
         was_cancelled = false
         aborted = false
+        handed_over = false
         held_open = false
+        started_at = Time.now
         started = monotonic_time
         deadline = started + timeout_sec
 
-        Open3.popen3(env || {}, *Shell.argv(command), chdir: cwd, pgroup: true) do |stdin, stdout, stderr, wait_thr|
-          stdin.close
-          readers = [stdout, stderr].map { |io| ProcessGroup::PipeReader.new(io, chunk: READ_CHUNK_BYTES) }
-
-          begin
-            until wait_thr.join(WAIT_SLICE_SEC)
-              if cancelled.call
-                was_cancelled = true
-                break
-              elsif monotonic_time > deadline
-                timed_out = true
-                break
-              end
+        id = TaskRuntime.generate_task_id
+        dir = File.join(logs_root, id)
+        FileUtils.mkdir_p(dir)
+        logs = [File.join(dir, STDOUT_LOG), File.join(dir, STDERR_LOG)]
+        exit_code_path = File.join(dir, TaskRuntime::EXIT_CODE_FILENAME)
+        begin
+          pid = spawn_logged(command, env: env, cwd: cwd, logs: logs, exit_code_path: exit_code_path)
+        rescue StandardError
+          FileUtils.rm_rf(dir)
+          raise
+        end
+        wait_thr = Process.detach(pid)
+        # A stop is done once the shell is reaped and nothing holds the logs:
+        # a child that ignores TERM keeps them, so the stop goes on to KILL.
+        stopped = -> { !wait_thr.alive? && !held?(logs) }
+
+        begin
+          until wait_thr.join(WAIT_SLICE_SEC)
+            if cancelled.call
+              was_cancelled = true
+              break
+            elsif monotonic_time > deadline
+              timed_out = true
+              break
+            end
+          end
+          # Ended right at the deadline: finished, not timed out.
+          timed_out = false if timed_out && !wait_thr.alive?
+          if timed_out && on_timeout == :task
+            text = hand_over(id: id, command: command, cwd: cwd, pid: pid, waiter: wait_thr, started_at: started_at,
+                             timeout_sec: timeout_sec, dir: dir, logs: logs, exit_code_path: exit_code_path)
+            if text
+              handed_over = true
+              raise HandedOver, text
             end
-            terminate_process_tree(wait_thr.pid) if timed_out || was_cancelled
-            wait_thr.join
-            status = wait_thr.value
-          rescue Exception # rubocop:disable Lint/RescueException -- kill the child, then re-raise
-            # An Interrupt (or anything else) mid-wait: the child is in its own
-            # process group, so it never saw the SIGINT. Kill it, or the reads
-            # below block until it closes stdout.
-            aborted = true
-            terminate_process_tree(wait_thr.pid)
-            raise
-          ensure
-            # The shell is gone, but a child it put in the background (`cmd &`)
-            # may still hold the pipes: wait for EOF a short grace only, then
-            # stop the command's process group and keep the output so far.
-            # Already stopped (timeout, Stop, an exception): no more checks.
-            stopping = timed_out || was_cancelled || aborted
-            outcome = await_readers(readers, until_time: monotonic_time + BACKGROUND_GRACE_SEC,
-                                             deadline: stopping ? nil : deadline,
-                                             cancelled: stopping ? -> { false } : cancelled)
-            unless outcome == :done
+            # Not adopted because it ended meanwhile: finished too.
+            timed_out = false unless wait_thr.alive?
+          end
+          terminate_process_tree(pid, stopped: stopped) if timed_out || was_cancelled
+          wait_thr.join
+          status = wait_thr.value
+        rescue HandedOver
+          raise
+        rescue Exception # rubocop:disable Lint/RescueException -- kill the child, then re-raise
+          # An Interrupt (or anything else) mid-wait: the child is in its own
+          # process group, so it never saw the SIGINT. Kill it.
+          aborted = true
+          terminate_process_tree(pid, stopped: stopped) if wait_thr.alive?
+          raise
+        ensure
+          unless handed_over
+            # The shell is gone; a child it put in the background may still
+            # hold the logs. Already stopped (timeout, Stop, an exception): no
+            # more checks.
+            unless timed_out || was_cancelled || aborted
+              outcome, held_open = background_check(pid, logs, deadline: deadline, cancelled: cancelled)
               was_cancelled = true if outcome == :cancelled
+              # Under on_timeout: task too: the shell is reaped, there is no
+              # leader left to hand over.
               timed_out = true if outcome == :timeout
-              stopped = stop_process_group(wait_thr.pid, readers)
-              held_open = (stopped ? :stopped : :detached) if outcome == :grace && !stopping
             end
-            readers.each { |reader| reader.finish(poll: STOP_POLL_INTERVAL_SEC, grace: STOP_GRACE_SEC) }
-            stdout_text, stderr_text = readers.map(&:text)
+            stdout_text, stderr_text = logs.map { |path| read_log(path) }
+            # A holder left running (setsid) still writes there: the dir
+            # stays, and the sweep removes it once no one holds the logs.
+            FileUtils.rm_rf(dir) unless held_open == :detached
           end
         end
 
@@ -147,38 +225,171 @@ module Samagotchi
       end
       private_class_method :run_command
 
-      # Waits for both readers to hit EOF until +until_time+, the overall
+      # @return [Integer] the pid of the shell, which leads its own group
+      def self.spawn_logged(command, env:, cwd:, logs:, exit_code_path:)
+        # Append mode: the shell and its background children share the open
+        # file, and each write lands at the end.
+        ios = logs.map { |path| File.open(path, "a") }
+        ios.each { |io| io.flock(File::LOCK_SH) }
+        ProcessGroup.spawn(env || {}, *Shell.argv(TaskRuntime.wrapped_shell_command(command, exit_code_path)),
+                           chdir: cwd, in: File::NULL, out: ios[0], err: ios[1])
+      ensure
+        ios&.each(&:close)
+      end
+      private_class_method :spawn_logged
+
+      # After the shell exited: a background child still holding the logs
+      # (`cmd &`) gets BACKGROUND_GRACE_SEC, then its group is stopped
+      # (BACKGROUND_HINT), unless it left the group (setsid) and holds them
+      # still (BACKGROUND_DETACHED_HINT, left running). One that holds none
+      # (`nohup cmd >f 2>&1 &`) is left alone, with no note.
+      # @return [Array(Symbol, Symbol|false)] the outcome (:done, :grace,
+      #   :timeout, :cancelled) and the hint (:stopped, :detached or false)
+      def self.background_check(pgid, logs, deadline:, cancelled:)
+        return [:done, false] unless held?(logs)
+
+        outcome = await_release(logs, until_time: monotonic_time + BACKGROUND_GRACE_SEC, deadline: deadline, cancelled: cancelled)
+        return [outcome, false] if outcome == :done
+
+        # Only the group: the leader's pid is reaped, so it may name another process.
+        ProcessGroup.stop(pgid, grace: STOP_GRACE_SEC, poll: STOP_POLL_INTERVAL_SEC, stopped: -> { !held?(logs) })
+        return [outcome, false] unless outcome == :grace
+
+        [outcome, held?(logs) ? :detached : :stopped]
+      end
+      private_class_method :background_check
+
+      # Whether some process still has one of +paths+ open from the spawn:
+      # a flock belongs to the open file description, so an exclusive try
+      # fails while any inherited copy is open (~0.2 ms; no lsof, no /proc).
+      def self.held?(paths)
+        paths.any? do |path|
+          File.open(path, "r") do |file|
+            got = file.flock(File::LOCK_EX | File::LOCK_NB)
+            file.flock(File::LOCK_UN) if got
+            !got
+          end
+        rescue Errno::ENOENT
+          false
+        end
+      end
+
+      # Polls the logs until no one holds them (+until_time+), the overall
       # +deadline+ or a Stop. Returns :done, :grace, :timeout or :cancelled.
-      def self.await_readers(readers, until_time:, deadline:, cancelled:)
+      def self.await_release(paths, until_time:, deadline:, cancelled:)
         loop do
-          return :done if readers.all?(&:done?)
+          return :done unless held?(paths)
           return :cancelled if cancelled.call
           return :timeout if deadline && monotonic_time > deadline
           return :grace if monotonic_time > until_time
 
-          readers.find { |r| !r.done? }&.wait(STOP_POLL_INTERVAL_SEC)
+          sleep(STOP_POLL_INTERVAL_SEC)
+        end
+      end
+      private_class_method :await_release
+
+      # The whole log when it is under execute.truncate_at_bytes, else its
+      # head and tail only, read by seek (never the whole 100 MB).
+      # @return [String, LogPreview]
+      def self.read_log(path)
+        return "" unless File.file?(path)
+
+        size = File.size(path)
+        return File.read(path) if size <= truncate_at_bytes
+
+        LogPreview.new(size, OutputGuardrails.head_tail_from_file(path: path, file_size: size, preview_bytes: preview_bytes))
+      end
+      private_class_method :read_log
+
+      # A command still running at the limit becomes a task (TaskRuntime.
+      # adopt) with the logs where they are; a back-pointer in its dir names
+      # the record for the sweep. nil when it can't be adopted (the caller
+      # then kills it as in :kill mode) or when it ended meanwhile (the
+      # caller then reads it as finished).
+      # @return [String, nil] the HANDED_OVER result
+      def self.hand_over(id:, command:, cwd:, pid:, waiter:, started_at:, timeout_sec:, dir:, logs:, exit_code_path:)
+        return nil unless waiter.alive?
+
+        record, error = TaskRuntime.adopt(id: id, command: command, cwd: cwd, pid: pid, started_at: started_at,
+                                          output_path: logs[0], stderr_path: logs[1], exit_code_path: exit_code_path)
+        return nil if error
+
+        File.write(File.join(dir, BACK_POINTER), File.expand_path(TaskRuntime.metadata_path(record["id"])))
+        write_missing_exit_code(waiter, exit_code_path)
+        # Ended between the deadline and the adopt: the record says how.
+        TaskRuntime.get_record(id) unless waiter.alive?
+        lines = [handed_over_line(timeout_sec), "task_id: #{id}",
+                 "Wait for it with task_wait (it may take minutes); task_stop stops it."]
+        { "stdout" => logs[0], "stderr" => logs[1] }.each do |label, path|
+          tail = TaskRuntime.output_tail_lines(path, HAND_OVER_TAIL_LINES)
+          lines << "#{label} (so far, tail):\n#{tail.chomp}" unless tail.empty?
+        end
+        lines.join("\n")
+      end
+      private_class_method :hand_over
+
+      # A command that set its own EXIT trap replaced the wrapper's, so it
+      # writes no exit_code: while this chi lives, the waiter's status fills
+      # it in (a refresh in between may still read "no exit code").
+      def self.write_missing_exit_code(waiter, path)
+        Thread.new do
+          Thread.current.report_on_exception = false
+          code = waiter.value&.exitstatus
+          File.write(path, code.to_s) if code && !File.exist?(path) && Dir.exist?(File.dirname(path))
+        end
+      end
+      private_class_method :write_missing_exit_code
+
+      # <state dir>/execute: every command's dir (LOGS_DIRNAME).
+      def self.logs_root = File.join(Paths.state_dir, LOGS_DIRNAME)
+
+      # Once per process, at the first command (sweep_leftovers).
+      def self.sweep_leftovers_once
+        return if @swept
+
+        @swept = true
+        sweep_leftovers
+      rescue StandardError
+        nil
+      end
+      private_class_method :sweep_leftovers_once
+
+      # Removes the dirs under logs_root no process holds any more: one
+      # without a back-pointer, older than ORPHAN_AGE_SEC (a chi that died
+      # mid-execute); one whose back-pointer's task.json is gone, or whose
+      # logs are older than RETENTION_DAYS (a handed-over task long done).
+      # A dir whose logs are still held is a command still running: kept.
+      # The records live in each project's tmp/tasks, which the state dir
+      # can't list: hence the back-pointer.
+      # @return [Array<String>] the dirs removed
+      def self.sweep_leftovers(now: Time.now)
+        Dir.glob(File.join(logs_root, "*")).select { |dir| File.directory?(dir) }.filter_map do |dir|
+          next unless leftover?(dir, now)
+
+          FileUtils.rm_rf(dir)
+          dir
         end
       end
-      private_class_method :await_readers
-
-      # The shell has exited (its pid is reaped) but its process group still
-      # has members holding the output: TERM the group, then KILL it. A process
-      # that left the group (setsid) is out of reach; the pipes get closed.
-      # True once the output was released, false if something still holds it.
-      # Only the group: the leader's pid is reaped, so it may name another process.
-      def self.stop_process_group(pgid, readers)
-        released = -> { readers.all?(&:done?) }
-        wait = ->(seconds) { readers.find { |r| !r.done? }&.wait(seconds) }
-        return true if ProcessGroup.stop(pgid, grace: STOP_GRACE_SEC, poll: STOP_POLL_INTERVAL_SEC, stopped: released, wait: wait)
-
-        await_readers(readers, until_time: monotonic_time + STOP_GRACE_SEC, deadline: nil, cancelled: -> { false }) == :done
+
+      def self.leftover?(dir, now)
+        logs = [File.join(dir, STDOUT_LOG), File.join(dir, STDERR_LOG)]
+        return false if held?(logs)
+
+        age = now - ([dir, *logs].filter_map { |path| File.mtime(path) if File.exist?(path) }.max || now)
+        pointer = File.join(dir, BACK_POINTER)
+        return age > ORPHAN_AGE_SEC unless File.file?(pointer)
+
+        !File.file?(File.read(pointer).strip) || age > RETENTION_DAYS * 24 * 3600
+      rescue SystemCallError
+        false
       end
-      private_class_method :stop_process_group
+      private_class_method :leftover?
 
       # The shell still runs: TERM its group (or the shell alone, if it left
-      # the group), then KILL it once the grace is over.
-      def self.terminate_process_tree(pid)
-        ProcessGroup.stop(pid, grace: STOP_GRACE_SEC, poll: STOP_POLL_INTERVAL_SEC, leader: true)
+      # the group), then KILL it once the grace is over, unless +stopped+
+      # (the shell reaped, the logs released) came first.
+      def self.terminate_process_tree(pid, stopped:)
+        ProcessGroup.stop(pid, grace: STOP_GRACE_SEC, poll: STOP_POLL_INTERVAL_SEC, leader: true, stopped: stopped)
       end
       private_class_method :terminate_process_tree
 
@@ -203,15 +414,29 @@ module Samagotchi
       end
       private_class_method :resolve_cwd
 
+      def self.truncate_at_bytes
+        OutputGuardrails.config_positive_int("execute.truncate_at_bytes", OutputGuardrails::DEFAULT_TRUNCATE_AT_BYTES)
+      end
+      private_class_method :truncate_at_bytes
+
+      def self.preview_bytes
+        OutputGuardrails.config_positive_int("execute.preview_bytes", OutputGuardrails::DEFAULT_PREVIEW_BYTES)
+      end
+      private_class_method :preview_bytes
+
+      # @param content [String, LogPreview] a log read whole, or its head
+      #   and tail (read_log)
       def self.output_block(label, content)
         return nil if content.nil? || content.empty?
 
-        truncate_at_bytes = OutputGuardrails.config_positive_int("execute.truncate_at_bytes", OutputGuardrails::DEFAULT_TRUNCATE_AT_BYTES)
-        preview_bytes = OutputGuardrails.config_positive_int("execute.preview_bytes", OutputGuardrails::DEFAULT_PREVIEW_BYTES)
         bytes = content.bytesize
-        return "#{label}:\n#{content}" if bytes <= truncate_at_bytes
+        return "#{label}:\n#{content}" if bytes <= truncate_at_bytes && !content.is_a?(LogPreview)
 
-        preview = OutputGuardrails.head_tail_from_string(content: content, preview_bytes: preview_bytes)
+        preview = if content.is_a?(LogPreview)
+                    content.preview
+                  else
+                    OutputGuardrails.head_tail_from_string(content: content, preview_bytes: preview_bytes)
+                  end
 
         [
           "#{label}:",
@@ -238,6 +463,29 @@ module Samagotchi
         )
       end
 
+      # A big log: its size, and the head and tail read by seek, which
+      # output_block shows as it shows a long string's.
+      class LogPreview
+        attr_reader :bytesize, :preview
+
+        def initialize(bytesize, preview)
+          @bytesize = bytesize
+          @preview = preview
+        end
+
+        def empty? = @bytesize.zero?
+      end
+
+      # Raised out of the wait with the hand-over's text: no read, no rm_rf.
+      class HandedOver < StandardError
+        attr_reader :text
+
+        def initialize(text)
+          @text = text
+          super("handed over")
+        end
+      end
+
       class CommandTimedOut < StandardError
         attr_reader :stdout, :stderr
 

data/lib/samagotchi/tools/task_get.rb

@@ -31,6 +31,8 @@ module Samagotchi
           "exit_code: #{record["exit_code"]}",
           "stop_reason: #{record["stop_reason"]}",
           "output_path: #{record.fetch("output_path")}",
+          # A handed-over execute's stderr has a log of its own.
+          ("stderr_path: #{record["stderr_path"]}" if record["stderr_path"]),
           (TaskRuntime::USER_STOP_NOTE if TaskRuntime.stopped_by_user?(record))
         ].compact.join("\n")
       end

data/lib/samagotchi/tools/task_runtime.rb

@@ -11,6 +11,10 @@ require_relative "shell"
 
 module Samagotchi
   module Tools
+    # Execute requires nothing from here at load, and P1b's Execute will
+    # require this file: autoload keeps the two from requiring each other.
+    autoload :Execute, File.expand_path("execute", __dir__)
+
     module TaskRuntime
       TASKS_DIR = File.join("tmp", "tasks")
       METADATA_FILENAME = "task.json"
@@ -97,6 +101,48 @@ module Samagotchi
         output_io&.close
       end
 
+      # A record for a command chi already runs: an execute still running at
+      # its limit, handed over as a task (Execute::HANDED_OVER). Its logs and
+      # exit_code file stay where execute put them, so the paths must be
+      # absolute; +stderr_path+ is a log of its own (task_wait tails it).
+      # The pid's start time is read here, at the hand-over, not at the
+      # spawn (a quick execute is spared the ps): read while execute's
+      # detach waiter is still running, so the pid is still its child.
+      # @param started_at [Time] when execute spawned the command
+      # @return [Array(Hash, nil), Array(nil, String)] the record, or an error
+      def adopt(id:, command:, cwd:, pid:, started_at:, output_path:, stderr_path:, exit_code_path:)
+        return [nil, "Error: invalid task id: #{id}"] unless TASK_ID_SHAPE.match?(id.to_s)
+        return [nil, "Error: task already exists: #{id}"] if File.exist?(metadata_path(id))
+
+        paths = { "output_path" => output_path, "stderr_path" => stderr_path, "exit_code_path" => exit_code_path }
+        relative = paths.find { |_, path| !File.absolute_path?(path.to_s) }
+        return [nil, "Error: #{relative[0]} must be absolute: #{relative[1]}"] if relative
+        return [nil, "Error: not a process chi can stop: #{pid.inspect}"] unless ProcessGroup.signalable?(pid)
+
+        started = started_at.utc.iso8601
+        record = {
+          "metadata_version" => METADATA_VERSION,
+          "id" => id,
+          "command" => command.to_s.strip,
+          "cwd" => File.expand_path(cwd.to_s.empty? ? Dir.pwd : cwd),
+          "workspace_root" => Dir.pwd,
+          "status" => "running",
+          "pid" => pid,
+          "pid_started" => ProcessGroup.start_time(pid),
+          "created_at" => started,
+          "started_at" => started,
+          "finished_at" => nil,
+          "exit_code" => nil,
+          "stop_reason" => nil,
+          **paths,
+          "handed_over_by" => Execute::NAME
+        }
+        write_record(record)
+        [record, nil]
+      rescue StandardError => e
+        [nil, "Error: #{e.message}"]
+      end
+
       def list_records
         ensure_tasks_dir
         task_dirs = Dir.glob(File.join(TASKS_DIR, "*"))
@@ -111,9 +157,10 @@ module Samagotchi
         end.compact.sort_by { |record| record["created_at"].to_s }.reverse
       end
 
-      # The tasks this conversation's task_create calls started that are
-      # still running. Only task_create results count: records are shared by
-      # every session in the worker's cwd, and task_list shows them all.
+      # The tasks this conversation started that are still running: its
+      # task_create calls and the executes it handed over at their limit.
+      # Only those results count: records are shared by every session in the
+      # worker's cwd, and task_list shows them all.
       # A native tool_response joins its calls' results with "---".
       # @return [Array<Hash>] {id:, command:}, oldest first
       def running_created_in(messages)
@@ -123,19 +170,53 @@ module Samagotchi
         end
       end
 
-      # The ids of every task this conversation's task_create calls started,
-      # whatever their status now, oldest first.
+      # The ids of every task this conversation started (task_create, a
+      # handed-over execute), whatever their status now, oldest first.
       # @return [Array<String>]
       def created_ids_in(messages)
         Array(messages).flat_map do |message|
           next [] unless (message[:role] || message["role"]).to_s == "tool_response"
 
           (message[:content] || message["content"]).to_s.split("\n\n---\n\n").filter_map do |block|
-            block[/\A\[task_create\]\ntask_id: (\S+)/, 1]
+            block[CREATED_BLOCK, 1] || handed_over_id(block)
           end
         end.uniq
       end
 
+      # A task_create result as the model gets it: its "[task_create]" lead,
+      # after the one "ran as: …" line a gate-corrected call has.
+      RAN_AS_LINE = "(?:ran as: [^\n]*\n)?"
+      CREATED_BLOCK = /\A#{RAN_AS_LINE}\[task_create\]\ntask_id: (\S+)/
+      CREATED_OUTPUT = /\A#{RAN_AS_LINE}(?:\[task_create\]\n)?task_id: (\S+)/
+
+      # The task id of a tool output that started a task, the "[tool]" lead
+      # optional (a UI's row has it or not): a task_create's, or an execute's
+      # handed_over_id. nil for any other tool or output.
+      # @return [String, nil]
+      def created_id(tool, text)
+        case tool
+        when "task_create" then text.to_s[CREATED_OUTPUT, 1]
+        when Execute::NAME then handed_over_id(text)
+        end
+      end
+
+      # The task id an execute's hand-over result names: anchored on its
+      # first line (Execute::HANDED_OVER), never a "task_id:" line anywhere,
+      # which the command's own output can print. The "[execute]" lead and
+      # a "ran as: …" line before it are optional. web/public/task_stop.js
+      # has the JS twin (handedOverId).
+      # @return [String, nil]
+      def handed_over_id(text)
+        text = text.to_s
+        # A regexp raises on bytes that aren't UTF-8 (a command's raw output).
+        text = text.scrub unless text.valid_encoding?
+        text[handed_over_pattern, 1]
+      end
+
+      def handed_over_pattern
+        @handed_over_pattern ||= /\A#{RAN_AS_LINE}(?:\[#{Execute::NAME}\]\n)?#{Regexp.escape(Execute::HANDED_OVER)} \d+s limit:[^\n]*\ntask_id: (\S+)/
+      end
+
       # A task id as task_create makes them: no path parts.
       TASK_ID = /\A[A-Za-z0-9_-]+\z/
 
@@ -295,14 +376,16 @@ module Samagotchi
         value.empty? ? Dir.pwd : File.expand_path(value)
       end
 
+      # The command with an EXIT trap set before it that writes the shell's
+      # exit status to +exit_code_path+. A prefix, not lines after the
+      # command: a trailing "\\" or "|" in the command would swallow those,
+      # and an `exit N` or a `set -e` failure would skip them. A command that
+      # sets its own EXIT trap replaces this one: no exit_code file then
+      # (the task ends failed with process_ended_without_exit_code, unless
+      # its starter's waiter writes the code: Execute's hand-over).
       def wrapped_shell_command(command, exit_code_path)
-        escaped_exit_code_path = Shellwords.escape(exit_code_path)
-        <<~SH
-          #{command}
-          status=$?
-          printf "%s" "$status" > #{escaped_exit_code_path}
-          exit "$status"
-        SH
+        write = %(printf "%s" "$?" > #{Shellwords.escape(exit_code_path)})
+        "trap #{Shellwords.escape(write)} EXIT\n#{command}\n"
       end
 
       def sanitized_spawn_env