lemans 1.3.2 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db9e22f2b2098371943cde85a366d70ff205b267719b67974484c5240752cd59
-  data.tar.gz: 707a10355b3fb212740b2839b51a1b346e8e6165a4aed5403f9c763045ffa6b8
+  metadata.gz: '08d62b21b79425a5c1e99867270b4cf2dd41ccb07f8ab7f5e5381a416170907f'
+  data.tar.gz: 802e7e3f665913b95a5409882f10b067f4118efe1e2de650a07dd93382f035af
 SHA512:
-  metadata.gz: f1472766d86a65544e6b525124fcd0afb828fa36c6a3c5ee1d4bb2d9f24972edb2d844abfdfc03735c6a9cd81b6bbda00b791f782b42cb7d523becdd01f764f1
-  data.tar.gz: 95d42dbaf57d92ace03b1bffdcf05316fd7244a5abb3fb74a5611ea15a4c8de4bdd86d6e9ab21b2b4bcf4606bce5ab11e5bf44fb372ae74c810d4c38b1c1b465
+  metadata.gz: fabad5219961a7514c44f888bd79713db444a1dc5b8b20c6153ae1e55c4252cc4065939bb6671a4e6511a1809f4b89da00ff2a642825348c184969fdf49a7c15
+  data.tar.gz: '08a08843cb79dcb1da14e27b1b11d2126fe0d8049dc7cdbcccb573c0c93d81f44613ed6f4e905ec23ab3e9f41e2f6eac44a49e9d520c50ebfca3f82a6d9f4f5f'

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [1.3.3] - 2026-09-11
+
+- Daytona: retry sandbox creation when the SDK gives up on a stalled start (the half-made sandbox is adopted or deleted first).
+- Miniswen: run local commands through `sh -c` (a missing command is exit 127, not a harness crash).
+- Miniswen: retry model calls for ~5 min instead of ~1, truncated responses included.
+- `lemans-remote run --launch-interval N` (default 3s) spaces sandbox launches; `--concurrency` is now a no-op.
+
 ## [1.3.2] - 2026-09-08
 
 - Miniswen: increase provider error max retry window to ~1 min.

data/exe/lemans-remote

@@ -16,7 +16,8 @@
 #   per task × model (--run-in-band for a single sandbox, --attempts N to
 #   repeat each task in N sandboxes); results are archived to the
 #   lemans-remote-runs volume. Add --sync to wait for a single sandbox
-#   and download into ./runs directly instead:
+#   and download into ./runs directly instead. Launches go out one at a
+#   time, --launch-interval seconds apart (default 3):
 #
 #     exe/lemans-remote run --bench ../ai-evals --task hello-world
 #     exe/lemans-remote run --bench ../ai-evals --model openrouter/z-ai/glm-5.2 --model openrouter/qwen/qwen3.5-coder --args="-k 2 -c 8"
@@ -32,7 +33,7 @@
 #
 #     exe/lemans-remote status [--history] [--running | --complete] [-W [INTERVAL]]
 #     exe/lemans-remote logs RUN_ID [--tail N]
-#     exe/lemans-remote pull-runs [RUN_ID ...] [--all] [--dry-run]
+#     exe/lemans-remote pull-runs [RUN_ID ...] [--match SUBSTR] [--all] [--dry-run]
 #     exe/lemans-remote drop-orphans [--min-age 10m]
 #     exe/lemans-remote clobber RUN_ID ... | --all
 #     exe/lemans-remote snapshots
@@ -440,10 +441,43 @@ module LemansRemote # :nodoc: all
       end
     end
 
-    def manifests(sandbox)
-      code = 'require "json"; puts Dir.glob("/vault/*/manifest.json", File::FNM_DOTMATCH)' \
-             ".filter_map { |f| JSON.parse(File.read(f)) rescue nil }.to_json"
-      response = sandbox.process.exec(command: "ruby -e #{Shellwords.escape(code)}", timeout: 120)
+    # The vault is an S3-backed FUSE mount: listing /vault is one request,
+    # but every /vault/<run>/manifest.json is another round-trip, so a plain
+    # glob over hundreds of runs outlives the exec timeout. List once, then
+    # read the manifests concurrently (and only the requested ones, if any).
+    MANIFEST_READERS = 32
+    MANIFESTS_TIMEOUT_SEC = 120
+
+    MANIFESTS_CODE = <<~'RUBY'
+      require "json"
+      only = JSON.parse(ARGV[0])
+      match = ARGV[1]
+      ids = Dir.children("/vault")
+      ids &= only if only
+      ids.select! { it.include?(match) } if match && !match.empty?
+      queue = Queue.new
+      ids.each { queue << it }
+      queue.close
+      found = Queue.new
+      readers = Array.new([ids.size, ENV.fetch("LEMANS_MANIFEST_READERS", "32").to_i].min) do
+        Thread.new do
+          while (id = queue.pop)
+            found << JSON.parse(File.read("/vault/#{id}/manifest.json")) rescue nil
+          end
+        end
+      end
+      readers.each(&:join)
+      list = []
+      list << found.pop until found.empty?
+      puts list.to_json
+    RUBY
+
+    def manifests(sandbox, only: nil, match: nil)
+      args = [ only&.to_a.to_json, match.to_s ].map { Shellwords.escape(it) }.join(" ")
+      response = sandbox.process.exec(
+        command: "LEMANS_MANIFEST_READERS=#{MANIFEST_READERS} ruby -e #{Shellwords.escape(MANIFESTS_CODE)} #{args}",
+        timeout: MANIFESTS_TIMEOUT_SEC
+      )
       raise "could not read the vault manifests: #{response.result}" unless response.exit_code.zero?
 
       body = response.result.to_s.strip
@@ -513,7 +547,7 @@ module LemansRemote # :nodoc: all
     CREATE_ATTEMPTS = 3
 
     def initialize(bench:, snapshot:, run_id:, tasks:, models:, extra_args:, extra_env:,
-                   timeout_sec:, runs_dir:, keep:, sync:, shell:)
+                   timeout_sec:, runs_dir:, keep:, sync:, shell:, launch_interval: 0)
       @bench = bench
       @snapshot = snapshot
       @run_id = run_id
@@ -526,6 +560,7 @@ module LemansRemote # :nodoc: all
       @keep = keep
       @sync = sync
       @shell = shell
+      @launch_interval = launch_interval
       @started_at = Time.now.utc.iso8601
     end
 
@@ -564,7 +599,7 @@ module LemansRemote # :nodoc: all
 
     # Daytona sometimes never starts a sandbox it accepted; the SDK gives up
     # after a minute and the half-made sandbox stays behind under this run's
-    # labels, listed as running until Daytona flags it, then stale forever.
+    # labels. A retry waits the launch interval first.
     def create_sandbox(volume = nil)
       attempt = 0
       begin
@@ -575,6 +610,7 @@ module LemansRemote # :nodoc: all
         raise if attempt >= CREATE_ATTEMPTS
 
         say :retry, "sandbox did not start (#{e.message}), attempt #{attempt + 1}/#{CREATE_ATTEMPTS}", :yellow
+        sleep @launch_interval if @launch_interval.positive?
         retry
       end
     end
@@ -872,7 +908,9 @@ module LemansRemote # :nodoc: all
     option :runs_dir, default: "runs", desc: "Sync mode: local directory to sync the results into"
     option :env, repeatable: true, desc: "Forward an extra host ENV variable by name"
     option :concurrency, type: :numeric, default: 4, aliases: "-C",
-                         desc: "Async mode: how many sandboxes to launch in parallel (1 to serialize)"
+                         desc: "Ignored: launches go out one at a time, --launch-interval apart (kept so older invocations still parse)"
+    option :launch_interval, type: :numeric, default: 3,
+                             desc: "Async mode: seconds between consecutive launches (0 to disable)"
     def run_bench
       bench = Lemans::Config.load_file(options[:bench])
 
@@ -899,7 +937,7 @@ module LemansRemote # :nodoc: all
       attempts = options[:attempts].to_i
       raise Thor::Error, "lemans-remote: --attempts must be at least 1" if attempts < 1
       raise Thor::Error, "lemans-remote: --attempts needs async mode — drop --sync" if options[:sync] && attempts > 1
-      raise Thor::Error, "lemans-remote: --concurrency must be at least 1" if options[:concurrency].to_i < 1
+      raise Thor::Error, "lemans-remote: --launch-interval must not be negative" if options[:launch_interval].to_f.negative?
 
       models = options[:model] || []
       jobs =
@@ -928,7 +966,7 @@ module LemansRemote # :nodoc: all
         elsif attempts > 1 then "#{jobs.size / attempts} task × model batch(es) × #{attempts} attempts"
         else "one per task × model"
         end
-      say_status :fanout, "#{jobs.size} sandboxes, #{fanout}, #{options[:concurrency].to_i} at a time"
+      say_status :fanout, "#{jobs.size} sandboxes, #{fanout}, one every #{options[:launch_interval]}s"
       failures, retried = launch_batches(bench, jobs, provisioner)
       finish_retry(planner, retried) if retry_groups
       say_status :detached, "`lemans-remote status` to watch, `lemans-remote pull-runs` to fetch results"
@@ -995,10 +1033,11 @@ module LemansRemote # :nodoc: all
     option :all, type: :boolean, default: false, desc: "Pull every completed run, even ones already pulled"
     option :dry_run, type: :boolean, default: false, desc: "Preview what would be downloaded without pulling anything"
     option :runs_dir, default: "runs", desc: "Local directory to sync the results into"
+    option :match, desc: "Only consider runs whose id contains this substring (e.g. a task name)"
     def pull_runs(*run_ids)
       pulled = []
       with_vault do |vault, sandbox|
-        manifests = vault.manifests(sandbox)
+        manifests = vault.manifests(sandbox, only: (run_ids if run_ids.any?), match: options[:match])
         known = manifests.map { it["run_id"] }
         targets =
           if run_ids.any?
@@ -1216,39 +1255,31 @@ module LemansRemote # :nodoc: all
         runs_dir: options[:runs_dir],
         keep: options[:keep],
         sync: options[:sync],
-        shell: shell
+        shell: shell,
+        launch_interval: options[:launch_interval].to_f
       ).call
     end
 
+    # One at a time, `interval` apart: a burst of creations stalls Daytona's
+    # scheduler.
     def launch_batches(bench, jobs, provisioner)
-      queue = Queue.new
-      jobs.each { queue << it }
-      failures = Queue.new
-      launched = Queue.new
+      interval = options[:launch_interval].to_f
+      failures = []
+      launched = []
 
-      threads = [ options[:concurrency].to_i, jobs.size ].min.times.map do
-        Thread.new do
-          loop do
-            batch, models, attempt, group =
-              begin
-                queue.pop(true)
-              rescue ThreadError
-                break
-              end
-            begin
-              launch_batch(bench, batch, models, provisioner, attempt:)
-              launched << group if group
-            rescue StandardError => e
-              label = batch.empty? ? "all" : batch.join(",")
-              label += "/a#{attempt}" if attempt
-              failures << "#{label} (#{e.message})"
-            end
-          end
+      jobs.each_with_index do |(batch, models, attempt, group), index|
+        sleep interval if index.positive? && interval.positive?
+        begin
+          launch_batch(bench, batch, models, provisioner, attempt:)
+          launched << group if group
+        rescue StandardError => e
+          label = batch.empty? ? "all" : batch.join(",")
+          label += "/a#{attempt}" if attempt
+          failures << "#{label} (#{e.message})"
         end
       end
-      threads.each(&:join)
 
-      [ Array.new(failures.size) { failures.pop }, Array.new(launched.size) { launched.pop } ]
+      [ failures, launched ]
     end
 
     def build_run_id(bench, tasks, models, attempt = nil)

data/lib/lemans/environments/daytona/retries.rb

@@ -7,6 +7,8 @@ module Lemans
     class Daytona
       # Another try for calls whose repeat is free. Only reads qualify: a
       # mutation may have landed server-side before its failure surfaced.
+      # Sandbox creation is the exception (Daytona#create_sandbox): what it
+      # half-made is found by label and dealt with before the repeat.
       module Retries
         # The snapshot service leaks the generated client's own error classes
         # instead of wrapping them, so both dialects have to be caught.

data/lib/lemans/environments/daytona.rb

@@ -9,12 +9,18 @@ module Lemans
     # Daytona sandboxes. Daytona builds images server-side into reusable content-named
     # snapshots and enforces the network policy itself.
     class Daytona < Environment
+      include Retries
+
       DEFAULT_BUILD_TIMEOUT = 600
 
       # Workspace tarballs ride uploads/downloads, so transfers get their own
       # budget through the SDK's streaming API instead of the global HTTP cap.
       TRANSFER_TIMEOUT = 900
 
+      CREATE_ATTEMPTS = 3
+      CREATE_RETRY_DELAY = 5
+      ADOPT_TIMEOUT = 180
+
       SDKTweaks.apply!
       # The SDK's typhoeus/libcurl transfers segfault the VM under concurrent
       # easy_perform calls; Faraday/Net::HTTP is pure Ruby, so transfers need
@@ -47,7 +53,7 @@ module Lemans
       end
 
       def start
-        @sandbox = client.create(create_params, on_snapshot_create_logs: @logger)
+        @sandbox = create_sandbox
         @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @shell = Shell.new(sandbox)
         self
@@ -116,6 +122,64 @@ module Lemans
 
       def client = self.class.client
 
+      # Under load Daytona accepts a sandbox, then stalls past the SDK's fixed
+      # minute. The one it half-made carries this trial's labels: adopt it if
+      # it comes up, delete it otherwise, and only then try again. Without
+      # labels there is nothing to find it by, so the failure stands.
+      def create_sandbox
+        attempt = 0
+        begin
+          attempt += 1
+          client.create(create_params, on_snapshot_create_logs: @logger)
+        rescue *Retries::SDK_ERRORS => e
+          raise if labels.empty? || !retryable?(e)
+
+          if (leftover = half_created(e))
+            return leftover if await_start(leftover)
+
+            reap!(leftover, e)
+          end
+          raise if attempt >= CREATE_ATTEMPTS
+
+          log "sandbox did not start (#{e.message}), attempt #{attempt + 1}/#{CREATE_ATTEMPTS}"
+          sleep CREATE_RETRY_DELAY
+          retry
+        end
+      end
+
+      # One create makes one sandbox; two under the same labels is not ours to sort out.
+      def half_created(error)
+        found = client.list(::Daytona::ListSandboxesQuery.new(labels: labels)).to_a
+        if found.size > 1
+          raise InfrastructureError, "#{could_not_start(error)}; #{found.map(&:id).join(", ")} all carry #{labels.inspect}"
+        end
+
+        found.first
+      rescue *Retries::SDK_ERRORS => e
+        raise InfrastructureError, "#{could_not_start(error)}; a sandbox may still be running under #{labels.inspect}, " \
+                                   "and listing them failed: #{e.message}"
+      end
+
+      # The SDK's wait fails at once on a dead sandbox.
+      def await_start(sandbox)
+        sandbox.wait_for_sandbox_start(ADOPT_TIMEOUT)
+        log "adopted sandbox #{sandbox.id}, which came up after the SDK gave up on it"
+        true
+      rescue *Retries::SDK_ERRORS => e
+        log "sandbox #{sandbox.id} did not come up on its own: #{e.message}"
+        false
+      end
+
+      def reap!(sandbox, error)
+        sandbox.delete
+      rescue *Retries::SDK_ERRORS => e
+        raise InfrastructureError, "#{could_not_start(error)}; #{sandbox.id} may still be running and could not be deleted: #{e.message}"
+      end
+
+      def could_not_start(error) = "daytona: could not start sandbox: #{error.message}"
+
+      def log(message) = @logger&.call("lemans: #{message}\n")
+
       # A sandbox inherits the snapshot's resources, so the profile's are
       # stamped into the snapshot at build time.
       def create_params

data/lib/lemans/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lemans
-  VERSION = "1.3.2"
+  VERSION = "1.3.3"
 end

data/lib/miniswen/agent.rb

@@ -513,7 +513,7 @@ module Miniswen
       body = e.response&.body.to_s
       detail = body.empty? ? e.message : "#{e.message}: #{body[0, 1000]}"
       raise InfrastructureError, "miniswen: the model call failed: #{detail}"
-    rescue Faraday::SSLError, Faraday::ConnectionFailed, Faraday::TimeoutError => e
+    rescue Faraday::SSLError, Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::ParsingError => e
       raise InfrastructureError, "miniswen: the model call failed: #{e.class}: #{e.message}"
     end
 

data/lib/miniswen/local.rb

@@ -9,8 +9,10 @@ module Miniswen
   class Local < Environment
     TIMEOUT_EXIT_CODE = 124
 
+    # Always through a shell: Ruby execs a metacharacter-free string directly,
+    # and a missing binary would then raise ENOENT here instead of exiting 127.
     def exec(command, timeout: nil, env: nil)
-      Open3.popen2e(env || {}, command, pgroup: true) do |stdin, io, wait_thr|
+      Open3.popen2e(env || {}, "sh", "-c", command, pgroup: true) do |stdin, io, wait_thr|
         stdin.close
         reader = Thread.new { io.read }
 

data/lib/miniswen/ruby_llm.rb

@@ -8,9 +8,10 @@ RubyLLM.configure do |config|
   config.logger = Logger.new(IO::NULL) unless ENV["MINISWEN_DEBUG"] == "1"
 end
 
-# About a minute of retries for egress blips (1, 2, 4, 8, 16, 32s plus jitter)
+# About five minutes of retries (1, 2, 4, ... 128s plus jitter): provider
+# outages and rate-limit windows outlast the minute this used to allow.
 RubyLLM.configure do |config|
-  config.max_retries = 6
+  config.max_retries = 8
   config.retry_interval = 1
 end
 
@@ -36,13 +37,13 @@ module Miniswen
     end
   end
 
-  # A handshake reset never sent the request, so retrying is as safe as the
-  # ConnectionFailed retries ruby_llm already does; it only lists SSL errors
-  # as fatal.
-  module RetryTransientSSL
-    def retry_exceptions = super + [ Faraday::SSLError ]
+  # ruby_llm lists SSL errors as fatal, but a handshake reset never sent the
+  # request; a 200 with a truncated JSON body is the same dropped connection
+  # one step later.
+  module RetryTransientFailures
+    def retry_exceptions = super + [ Faraday::SSLError, Faraday::ParsingError ]
   end
 end
 
 RubyLLM::Providers::OpenRouter.prepend(Miniswen::VerbatimReasoningDetails)
-RubyLLM::Connection.prepend(Miniswen::RetryTransientSSL)
+RubyLLM::Connection.prepend(Miniswen::RetryTransientFailures)

data/lib/miniswen/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miniswen
-  VERSION = "1.3.2"
+  VERSION = "1.3.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lemans
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.3.3
 platform: ruby
 authors:
 - Svyatoslav Kryukov
@@ -53,6 +53,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.203'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement