samagotchi 0.91.0 → 0.92.0

data/lib/samagotchi/bundle_command/try/run.rb

@@ -2,10 +2,20 @@
 
 require "open3"
 require "rbconfig"
+require_relative "../../process_group"
 
 module Samagotchi
   class BundleCommand
     module Try
+      # The sandboxed child chi of `chi bundle try`, in a process group of
+      # its own (ProcessGroup). A --timeout sends it INT (a Ctrl-C: the turn
+      # is saved), then after INTERRUPT_GRACE ProcessGroup.stop stops the
+      # group and the groups its processes started. After any exit the
+      # group is stopped only while something of the child's is still in it
+      # (#stop_stragglers): a group signal to a leader reaped with nothing
+      # left would reach whatever reuses its pid. What left the group and
+      # its parent (a delegate's worker, a task once the child exited) is
+      # Sandbox#stop_leftovers', by the records in the sandbox.
       class Run
         Result = Data.define(:exit_status, :stdout, :stderr, :session_id, :timed_out)
         INTERRUPT_GRACE = 5
@@ -30,18 +40,19 @@ module Samagotchi
               streams = [stdout, stderr]
               stdin.close
               readers = streams.map { |stream| Thread.new { read_stream(stream) } }
-              attach_child(trap_state, pid)
+              attach_child(trap_state, pid, wait)
               timed_out = false
               status = wait_for(wait, timeout)
               if status == :timeout
                 timed_out = true
-                signal_group(pid, "INT")
+                ProcessGroup.signal(pid, "INT")
                 status = wait_for(wait, INTERRUPT_GRACE)
                 if status == :timeout
-                  terminate_group(pid)
+                  ProcessGroup.stop(pid, grace: GROUP_TERM_GRACE, poll: POLL_INTERVAL, stopped: -> { !wait.alive? })
                   status = wait.value
                 end
               end
+              trap_state[:pid] = nil
               out, err = finish_readers(readers, streams)
               code = timed_out ? CLI::Exit::FAILED : process_exit_status(status)
               Result.new(exit_status: code, stdout: out, stderr: err, session_id: err[/^Session: (\S+)/, 1],
@@ -50,7 +61,8 @@ module Samagotchi
           rescue SystemCallError => e
             raise Sandbox::Error, "chi bundle try: couldn't run chi: #{e.message}"
           ensure
-            terminate_group(pid) if pid
+            trap_state[:pid] = nil
+            stop_stragglers(pid) if pid
             close_streams(streams)
             stop_readers(readers)
             restore_traps(old_traps)
@@ -79,12 +91,13 @@ module Samagotchi
         end
 
         def trap_forwarding
-          state = { pid: nil, pending: nil }
+          state = { pid: nil, wait: nil, pending: nil }
           old = {}
           %w[INT TERM].each do |signal|
             old[signal] = Signal.trap(signal) do
               if state[:pid]
-                signal_group(state[:pid], signal)
+                # Not once its waiter reaped it: that group may be gone.
+                ProcessGroup.signal(state[:pid], signal) if state[:wait]&.alive?
               else
                 state[:pending] ||= signal
               end
@@ -93,27 +106,32 @@ module Samagotchi
           [state, old]
         end
 
-        def attach_child(state, pid)
+        def attach_child(state, pid, wait)
+          state[:wait] = wait
           state[:pid] = pid
           pending = state[:pending]
           state[:pending] = nil
-          signal_group(pid, pending) if pending
+          ProcessGroup.signal(pid, pending) if pending
         end
 
         def restore_traps(old)
           old.each { |signal, handler| Signal.trap(signal, handler) }
         end
 
-        def signal_group(pid, signal)
-          Process.kill(signal, -pid)
-        rescue Errno::ESRCH
-          nil
-        end
+        # Stops what is left in the child's group once the child (its
+        # leader) is reaped: a descendant still holding its pipes. Only
+        # while one is in it: no process left means no group, and the pid
+        # may be reused by now. While a member is left the group's id
+        # can't be reused; the reaped leader's pid can, so a group whose
+        # member is the pid itself is someone else's.
+        def stop_stragglers(pid)
+          return unless ProcessGroup.signalable?(pid) && ProcessGroup.group_alive?(pid)
+
+          members = ProcessGroup.process_table.select { |_pid, _ppid, group| group == pid }.map(&:first)
+          return if members.empty? || members.include?(pid)
 
-        def terminate_group(pid)
-          signal_group(pid, "TERM")
-          sleep(GROUP_TERM_GRACE)
-          signal_group(pid, "KILL")
+          ProcessGroup.stop(pid, grace: GROUP_TERM_GRACE, poll: POLL_INTERVAL,
+                                 stopped: -> { !ProcessGroup.group_alive?(pid) })
         end
 
         def read_stream(stream)

data/lib/samagotchi/bundle_command/try/sandbox.rb

@@ -6,11 +6,18 @@ require "rbconfig"
 require "tmpdir"
 require "yaml"
 require_relative "../../config"
+require_relative "../../child_env"
 require_relative "../../cli/exit"
-require_relative "../../installed_gem"
+require_relative "../../memory_bundle/system_bundle"
 require_relative "../../memory_bundle"
 require_relative "../../memory_bundle/shas"
+require_relative "../../owner_lock"
+require_relative "../../process_group"
+require_relative "../../sandbox_layout"
+require_relative "../../session"
 require_relative "../../session_manager"
+require_relative "../../tools/task_record"
+require_relative "../../tools/task_runtime"
 
 module Samagotchi
   class BundleCommand
@@ -19,18 +26,46 @@ module Samagotchi
         LIB_DIR = File.expand_path("../../..", __dir__)
         CHI = File.expand_path("../../../../bin/chi", __dir__)
         SHIPPED_DIR = MemoryBundle::SourceNormalizer::SHIPPED_DIR
+        # The config.yml sections the sandbox drops (and their SAMAGOTCHI_<SECTION>_* env forms); context:
+        # loses only its sources: (#write_config).
         CONFIG_DENYLIST = %w[
           bundles hooks guardrails memories append_system_prompt recap session log web update kitty tmux broadcast
-          context_sources
         ].freeze
         CONFIG_DENYLIST_ENV_PREFIXES = CONFIG_DENYLIST.map do |section|
           prefix = if section == "guardrails"
-                     SessionManager::GUARDRAILS_ENV_PREFIX.delete_suffix("_")
+                     ChildEnv::GUARDRAILS_PREFIX.delete_suffix("_")
                    else
                      "SAMAGOTCHI_#{section.upcase}"
                    end
           prefix
         end.freeze
+        # `chi bundle install PATH --force` for each PATH in ARGV, in one
+        # process (bin/chi's start, once): an install that fails prints
+        # what it printed and exits with its status; the ones before it
+        # print nothing.
+        INSTALL_SCRIPT = <<~RUBY
+          require "stringio"
+          require "samagotchi/utf8_default"
+          Samagotchi::Utf8Default.apply!
+          require "samagotchi/config"
+          Samagotchi::ConfigFile.load!
+          require "samagotchi"
+          require "samagotchi/bundle_command"
+          ARGV.each do |path|
+            out = StringIO.new
+            err = StringIO.new
+            status = Samagotchi::BundleCommand.new(["install", path, "--force"], stdout: out, stderr: err).run
+            next if status.zero?
+
+            $stdout.write(out.string)
+            $stderr.write(err.string)
+            exit(status)
+          end
+        RUBY
+        # How long a leftover worker or task gets after TERM (a worker saves
+        # its session) before ProcessGroup.stop KILLs it.
+        LEFTOVER_GRACE = 3.0
+        LEFTOVER_POLL = 0.05
         class Error < StandardError
           attr_reader :status
 
@@ -40,7 +75,7 @@ module Samagotchi
           end
         end
 
-        attr_reader :root, :config_home, :state_home, :work_dir, :manifest
+        attr_reader :root, :config_home, :state_home, :work_dir
 
         def initialize(path, bundle_settings: {}, cost_cap: 1, cost_cap_override: false)
           @path = File.realpath(path)
@@ -48,12 +83,11 @@ module Samagotchi
           @cost_cap = cost_cap
           @cost_cap_override = cost_cap_override
           read_manifest
-          @root = File.realpath(Dir.mktmpdir("chi-bundle-try-"))
-          @config_home = File.join(root, "config")
-          @state_home = File.join(root, "state")
-          @work_dir = File.join(root, "work")
-          [config_home, state_home, work_dir].each { |dir| FileUtils.mkdir_p(dir) }
-          @work_dir = File.realpath(@work_dir)
+          @layout = SandboxLayout.create("chi-bundle-try-")
+          @root = @layout.root
+          @config_home = @layout.config_home
+          @state_home = @layout.state_home
+          @work_dir = @layout.work_dir
           write_config
           @child_env = build_child_env
         rescue StandardError
@@ -63,20 +97,43 @@ module Samagotchi
 
         def child_env = @child_env.dup
 
+        # Installs the tried bundle, then the shipped guardrails and
+        # cost-cap (unless it is one of them), each as `chi bundle install
+        # PATH --force` in one child chi with the sandbox's environment.
+        # @return [MemoryBundle::Manifest]
+        # @raise [Error] what the first install that failed printed
         def install
-          install_path(@path)
-          %w[guardrails cost-cap].each do |name|
-            next if @manifest.name == name
-
-            install_path(File.join(SHIPPED_DIR, name))
+          paths = [@path] + %w[guardrails cost-cap].reject { |name| @manifest.name == name }
+                                                     .map { |name| File.join(SHIPPED_DIR, name) }
+          out, err, status = Open3.capture3(child_env, RbConfig.ruby, "-I", LIB_DIR, "-e", INSTALL_SCRIPT, *paths,
+                                            chdir: work_dir)
+          unless status.success?
+            message = [out, err].reject(&:empty?).join("\n").strip
+            raise Error, "chi bundle try: #{message.empty? ? "bundle install failed" : message}"
           end
           @manifest
         end
 
         def cleanup
+          stop_leftovers
           FileUtils.rm_rf(root) if root && File.exist?(root)
         end
 
+        # Stops what the run started that outlives its child chi: the
+        # workers of its delegates (they leave the child's process tree,
+        # SessionManager.detach_from_spawner) and its tasks (a group of
+        # their own each, reparented once the child exits). Found by their
+        # records in the sandbox, so nothing outside it: a session's lock
+        # held by a group leader, a running task's record whose pid still
+        # leads its group and started when the record says (#same_process?). Workers
+        # first: one may start a task as it stops.
+        def stop_leftovers
+          return unless root && File.directory?(root)
+
+          leftover_workers.each { |pid| stop_group(pid) }
+          leftover_tasks.each { |pid| stop_group(pid) }
+        end
+
         private
 
         def write_config
@@ -92,9 +149,7 @@ module Samagotchi
           data["bundles"][@manifest.name] ||= {}
           data["bundles"][@manifest.name].merge!(@bundle_settings)
           data["bundles"]["cost-cap"]["cap"] = @cost_cap if @cost_cap_override
-          path = File.join(config_home, "samagotchi", "config.yml")
-          FileUtils.mkdir_p(File.dirname(path))
-          File.write(path, YAML.dump(data))
+          @layout.write_config(data)
         end
 
         def read_manifest
@@ -121,36 +176,60 @@ module Samagotchi
           raise Error, "chi bundle try: manifest.yml is not valid YAML (#{e.message})"
         end
 
-        def install_path(path)
-          out, err, status = Open3.capture3(child_env, RbConfig.ruby, "-I", LIB_DIR, CHI,
-                                            "bundle", "install", path, "--force", chdir: work_dir)
-          return if status.success?
+        def leftover_workers
+          sessions = File.join(state_home, Session::STATE_SUBDIR)
+          Dir.glob(File.join(sessions, "*", OwnerLock::FILE)).filter_map do |lock|
+            pid = OwnerLock.owner(File.dirname(lock))&.pid
+            pid if ProcessGroup.leader?(pid)
+          end
+        end
+
+        def leftover_tasks
+          pattern = File.join(work_dir, Tools::TaskRuntime::TASKS_DIR, "*", Tools::TaskRuntime::METADATA_FILENAME)
+          Dir.glob(pattern).filter_map do |file|
+            record = Tools::TaskRecord.from_file(JSON.parse(File.read(file)))
+            record.pid if record&.running? && same_process?(record)
+          rescue JSON::ParserError, SystemCallError
+            nil
+          end
+        end
+
+        # Stricter than TaskRuntime.chis_group?: a record with no start time
+        # (a hand-over not adopted yet) or a start time that can't be read
+        # now is skipped, never a group whose leader may reuse the pid.
+        def same_process?(record)
+          started = record.pid_started
+          return false unless started && ProcessGroup.signalable?(record.pid)
+
+          ProcessGroup.start_time(record.pid) == started && ProcessGroup.leader?(record.pid)
+        end
 
-          message = [out, err].reject(&:empty?).join("\n").strip
-          raise Error, "chi bundle try: #{message.empty? ? "bundle install failed" : message}"
+        def stop_group(pid)
+          ProcessGroup.stop(pid, grace: LEFTOVER_GRACE, poll: LEFTOVER_POLL)
+        rescue ArgumentError, SystemCallError
+          nil
         end
 
+        # The user's environment (their hosts and keys) minus what no child
+        # chi takes from its spawner (ChildEnv), the worker markers, this
+        # chi's log and test settings, and the env forms of the config
+        # sections the sandbox drops.
         def build_child_env
-          env = ENV.to_h
-          SessionManager::SESSION_MARKER_ENV.each { |key| env.delete(key) }
+          env = ChildEnv.scrub(ENV.to_h)
           env.delete("SAMAGOTCHI_LOG_FILE")
           env.delete("SAMAGOTCHI_LOG_DISABLE")
-          env.delete("SAMAGOTCHI_SKIP_SYSTEM_BUNDLE")
+          env.delete(MemoryBundle::SystemBundle::SKIP_ENV)
           env.delete("SAMAGOTCHI_ENV") if env["SAMAGOTCHI_ENV"] == "test"
-          env.delete("SAMAGOTCHI_INSTALLED_VERSION")
           env.delete_if do |key, _|
-            key.start_with?(SessionManager::WORKER_ENV_PREFIX) || key == SessionManager::BOOT_FALLBACK_ENV ||
+            key.start_with?(ChildEnv::WORKER_PREFIX) || key == SessionManager::BOOT_FALLBACK_ENV ||
               CONFIG_DENYLIST_ENV_PREFIXES.any? { |prefix| key == prefix || key.start_with?("#{prefix}_") }
           end
           env.merge!(
             "HOME" => Dir.home,
             "SAMAGOTCHI_KEYCHAIN" => "on",
-            "XDG_CONFIG_HOME" => config_home,
-            "XDG_STATE_HOME" => state_home,
-            "SAMAGOTCHI_RECAP_ENABLED" => "false",
-            "SAMAGOTCHI_CACHE_WARMUP" => "off"
+            **@layout.env
           )
-          env.merge!(InstalledGem.unbundled_env_changes(env)) if InstalledGem.spec
+          env.merge!(ChildEnv.unbundled(env))
           env
         end
       end

data/lib/samagotchi/bundle_command/try/trace.rb

@@ -2,7 +2,10 @@
 
 require "json"
 require_relative "../../analytics_file"
+require_relative "../../bridge/card_store"
+require_relative "../../session"
 require_relative "../../session_metrics"
+require_relative "../../tool_response"
 
 module Samagotchi
   class BundleCommand
@@ -15,25 +18,23 @@ module Samagotchi
 
       class TraceReader
         DENIAL_TEXT = "No one to approve it (non-interactive run)."
-        SESSION_ID = /\A[A-Za-z0-9_-]+\z/
         STDERR_LIMIT = 4000
 
         def initialize(state_home:, session_id:)
           @state_home = state_home
-          @session_id = session_id if session_id.to_s.match?(SESSION_ID)
+          @session_id = session_id if Session.valid_id?(session_id)
         end
 
         def read(exit_code:, answer:, stderr:, sandbox:, timed_out: false)
           return empty_trace(exit_code, answer, stderr, sandbox, timed_out) unless @session_id
 
-          sessions_dir = File.join(@state_home, "samagotchi", "sessions")
-          session_path = File.join(sessions_dir, "#{@session_id}.json")
-          session_dir = File.join(sessions_dir, @session_id)
-          session = json_file(session_path) || {}
+          sessions_dir = File.join(@state_home, Session::STATE_SUBDIR)
+          session_dir = Session.session_dir(@session_id, state_dir: sessions_dir)
+          session = json_file(Session.session_file(@session_id, state_dir: sessions_dir)) || {}
           messages = Array(session["messages"]).select { |message| message.is_a?(Hash) }
           analytics = AnalyticsFile.read(session_dir) || {}
           entries = card_entries(session_dir)
-          tool_rows, denied = tools(messages, analytics, timed_out)
+          tool_rows, denied = tools(messages, analytics)
           summary = SessionMetrics.saved_summary(session_dir)
           cost, estimate = cost_summary(summary)
           questions = entries.select { |entry| entry["type"] == "question" }
@@ -62,7 +63,11 @@ module Samagotchi
                     timed_out: timed_out)
         end
 
-        def tools(messages, analytics, timed_out)
+        # One row per saved call. A turn a --timeout interrupt cut (a
+        # Ctrl-C) saves its calls too, the one it cut short answered as
+        # interrupted (ToolResponse.interrupted): a record has no row of its
+        # own, so no call shows twice.
+        def tools(messages, analytics)
           records = Array(analytics["tool_records"]).select { |record| record.is_a?(Hash) }
                                                     .to_h do |record|
             [[record["iteration"].to_i, record["call_index"].to_i], record]
@@ -86,6 +91,7 @@ module Samagotchi
               is_denied = response && response["content"].to_s.include?(DENIAL_TEXT)
               denied << index if is_denied
               status = is_denied ? "denied" : record&.dig("status").to_s
+              status = "interrupted" if status.empty? && response && ToolResponse.interrupted_text?(response["content"])
               status = "unknown" if status.empty?
               {
                 index: index,
@@ -96,24 +102,9 @@ module Samagotchi
               }
             end
           end
-          rows += unsaved_rows(records, iteration, index) if timed_out
           [rows, denied]
         end
 
-        # A turn cut by a --timeout interrupt saves only the messages before
-        # it (a Ctrl-C), while its tool records stay: the records past the
-        # last saved model message are its calls, without their arguments.
-        # Only then: a normal run's loop also counts iterations it saves no
-        # model message for (a cut's retry, an empty-answer nudge).
-        def unsaved_rows(records, saved_iterations, index)
-          records.select { |(iteration, _), _| iteration > saved_iterations }.sort.map do |_, record|
-            index += 1
-            status = record["status"].to_s
-            { index: index, name: record["called_as"] || record["tool"] || "tool", args: "",
-              status: status.empty? ? "unknown" : status, duration_ms: record["duration_ms"].to_i }
-          end
-        end
-
         def call_name(call)
           function = call["function"].is_a?(Hash) ? call["function"] : {}
           call["name"] || function["name"] || "tool"
@@ -132,10 +123,13 @@ module Samagotchi
           args.to_s.gsub(/\s+/, " ").strip[0, 200]
         end
 
+        # The listing's placement fields: nothing to place in a one-turn run.
+        LISTING_FIELDS = %w[turns_since current earlier].freeze
+
+        # The saved cards as the web reads them (CardStore.saved), with
+        # string keys and values, as the JSON trace prints them.
         def card_entries(session_dir)
-          data = json_file(File.join(session_dir, "cards.json"))
-          entries = data.is_a?(Hash) ? data["entries"] : []
-          Array(entries).select { |entry| entry.is_a?(Hash) }
+          JSON.parse(JSON.generate(Bridge::CardStore.saved(session_dir))).map { |entry| entry.except(*LISTING_FIELDS) }
         end
 
         def unanswerable?(question)