lemans 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d451c9e558e461c51e7be30bdc17d9921a0ae8f028d82946dd5c0538620c5c31
-  data.tar.gz: 3ff346d1263965339b9e51b51eaefade1fdd581048cc6b21c876836503cef3c8
+  metadata.gz: 6ce349f204cd0dc573a18932210dd5298941fe097f24576df1e318b29ce7152a
+  data.tar.gz: c9113c9bc9dbc4cf1f92d422b3ff929b20420e769828f55585f2a3eead34aa93
 SHA512:
-  metadata.gz: e8d4b6027bcb1734cdfb4ca6c4f9797577014dc138189bbafe99fd8476cd07ecec52dbe96c5ad6107f44776b500d92a687d790d202699199ecb90b130522ecf8
-  data.tar.gz: e0a3ab512fad47cc54b2057d06af7ddd7c2cc27d3b6d7c81ac3284c49ba4109900171f871907f163eb89180d0b82312a4a9039710c13ca24a084a3efc9b6041f
+  metadata.gz: 89ecd01b291a2d75475f346e8e780d87e104139e0ca34f48dde192325ec2ed593920e0e63f8b41a26b9e13144052e6bfe841831bc5d40a4432a0cd98d3fc39fc
+  data.tar.gz: 84b2bceb8d5b7c60b0e6513bb05b801d38081f8f59c55e96338a2a47d660c0521a31d224c365e0b694581f4e600552e4e9e390f3ccd6130d7fc61c8a9897cce2

data/exe/lemans-remote

@@ -13,8 +13,9 @@
 #     exe/lemans-remote provision
 #
 #   Fire-and-forget a bench run on remote Daytona sandboxes — one sandbox
-#   per task (--run-in-band for a single sandbox); results are archived to
-#   the lemans-remote-runs volume. Add --sync to wait for a single sandbox
+#   per task (--run-in-band for a single sandbox, --attempts N to repeat
+#   each task in N sandboxes); results are archived to the
+#   lemans-remote-runs volume. Add --sync to wait for a single sandbox
 #   and download into ./runs directly instead:
 #
 #     exe/lemans-remote run --bench ../ai-evals --task hello-world
@@ -22,9 +23,11 @@
 #
 #   Then watch, fetch, and clean up:
 #
-#     exe/lemans-remote status [--history]
-#     exe/lemans-remote pull-runs [RUN_ID ...] [--all]
+#     exe/lemans-remote status [--history] [--running | --complete]
+#     exe/lemans-remote pull-runs [RUN_ID ...] [--all] [--dry-run]
 #     exe/lemans-remote clobber RUN_ID ... | --all
+#     exe/lemans-remote snapshots
+#     exe/lemans-remote deprovision [--name SNAPSHOT]
 #
 #   Credentials come from the host ENV: DAYTONA_API_KEY (or DAYTONA_TOKEN),
 #   OPENROUTER_API_KEY, LEMANS_PROVIDER_ORDER; forward extras with --env KEY.
@@ -47,7 +50,7 @@ require "timeout"
 require "tmpdir"
 require "openssl"
 
-module LemansRemote
+module LemansRemote # :nodoc: all
   LABEL = "lemans-remote"
   RUN_ID_LABEL = "lemans-remote/run-id"
   STATUS_LABEL = "lemans-remote/status"
@@ -111,7 +114,7 @@ module LemansRemote
       name
     end
 
-    def drop(target = name)
+    def drop(target = name) # rubocop:disable Naming/PredicateMethod
       existing = find(target)
       return false unless existing
 
@@ -348,7 +351,7 @@ module LemansRemote
     end
 
     def manifests(sandbox)
-      code = 'require "json"; puts Dir.glob("/vault/*/manifest.json")' \
+      code = 'require "json"; puts Dir.glob("/vault/*/manifest.json", File::FNM_DOTMATCH)' \
              ".filter_map { |f| JSON.parse(File.read(f)) rescue nil }.to_json"
       response = sandbox.process.exec(command: "ruby -e #{Shellwords.escape(code)}", timeout: 120)
       raise "could not read the vault manifests: #{response.result}" unless response.exit_code.zero?
@@ -357,6 +360,14 @@ module LemansRemote
       body.empty? ? [] : JSON.parse(body)
     end
 
+    def archive_size(sandbox, run_id)
+      remote = "/vault/#{run_id}/runs.tar.gz"
+      response = sandbox.process.exec(command: "du -h #{Shellwords.escape(remote)}", timeout: 60)
+      return nil unless response.exit_code.zero?
+
+      response.result.to_s.split.first
+    end
+
     def download_runs(sandbox, run_id, runs_dir)
       remote = "/vault/#{run_id}/runs.tar.gz"
       probe = sandbox.process.exec(command: "test -f #{Shellwords.escape(remote)}", timeout: 60)
@@ -384,7 +395,7 @@ module LemansRemote
     def mark_pulled(sandbox, run_id)
       code = 'require "json"; require "time"; ' \
              'path = File.join("/vault", ARGV[0], "manifest.json"); ' \
-             'data = JSON.parse(File.read(path)); ' \
+             "data = JSON.parse(File.read(path)); " \
              'data["pulled_at"] = Time.now.utc.iso8601; ' \
              "File.write(path, JSON.pretty_generate(data))"
       sandbox.process.exec(command: "ruby -e #{Shellwords.escape(code)} #{Shellwords.escape(run_id)}", timeout: 60)
@@ -537,7 +548,7 @@ module LemansRemote
     def run_metadata(sandbox)
       {
         "run_id" => @run_id,
-        "bench" => @bench.root.basename.to_s,
+        "bench" => @bench.root.expand_path.basename.to_s,
         "tasks" => @tasks,
         "models" => @models,
         "args" => @extra_args,
@@ -722,6 +733,7 @@ module LemansRemote
     option :sync, type: :boolean, default: false, desc: "Wait for the run and download the results directly"
     option :run_in_band, type: :boolean, default: false, aliases: %w[--runInBand],
                          desc: "Async mode: run all tasks in one sandbox instead of one sandbox per task"
+    option :attempts, type: :numeric, default: 1, desc: "Async mode: repeat each task batch N times, one sandbox per attempt"
     option :keep, type: :boolean, default: false, desc: "Sync mode: leave the sandbox around for debugging"
     option :runs_dir, default: "runs", desc: "Sync mode: local directory to sync the results into"
     option :env, repeatable: true, desc: "Forward an extra host ENV variable by name"
@@ -732,23 +744,28 @@ module LemansRemote
       raise Thor::Error, "lemans-remote: no such task(s): #{unknown.join(", ")}" if unknown.any?
 
       provisioner = build_provisioner
-      unless provisioner.provisioned?
-        raise Thor::Error, "lemans-remote: snapshot #{provisioner.name} not found — run `lemans-remote provision` first"
-      end
+      raise Thor::Error, "lemans-remote: snapshot #{provisioner.name} not found — run `lemans-remote provision` first" unless provisioner.provisioned?
+
+      attempts = options[:attempts].to_i
+      raise Thor::Error, "lemans-remote: --attempts must be at least 1" if attempts < 1
+      raise Thor::Error, "lemans-remote: --attempts needs async mode — drop --sync" if options[:sync] && attempts > 1
 
       models = options[:model] || []
       batches = plan_batches(bench, tasks)
+      jobs = batches.flat_map { |batch| (1..attempts).map { [batch, attempts > 1 ? _1 : nil] } }
       Vault.ensure unless options[:sync]
 
-      if batches.size == 1
-        exit_code = launch_batch(bench, batches.first, models, provisioner)
+      if jobs.size == 1
+        batch, attempt = jobs.first
+        exit_code = launch_batch(bench, batch, models, provisioner, attempt:)
         say_status :detached, "`lemans-remote status` to watch, `lemans-remote pull-runs` to fetch results" unless options[:sync]
         exit exit_code unless exit_code.zero?
         return
       end
 
-      say_status :fanout, "#{batches.size} sandboxes, one per task"
-      failures = launch_batches(bench, batches, models, provisioner)
+      fanout = attempts > 1 ? "#{batches.size} task batch(es) × #{attempts} attempts" : "one per task"
+      say_status :fanout, "#{jobs.size} sandboxes, #{fanout}"
+      failures = launch_batches(bench, jobs, models, provisioner)
       say_status :detached, "`lemans-remote status` to watch, `lemans-remote pull-runs` to fetch results"
       raise Thor::Error, "lemans-remote: failed to launch: #{failures.join("; ")}" if failures.any?
     rescue Lemans::ConfigError, RuntimeError => e
@@ -757,6 +774,8 @@ module LemansRemote
 
     desc "status", "List lemans-remote runs (--history adds completed runs whose sandboxes are gone)"
     option :history, type: :boolean, default: false, desc: "Also read the vault manifests (spins a short-lived helper sandbox)"
+    option :running, type: :boolean, default: false, desc: "Only show runs still in flight"
+    option :complete, type: :boolean, default: false, desc: "Only show finished runs (success, failed, or stale)"
     def status
       rows = Fleet.rows.map { [_1.run_id, _1.status, _1.started_at || "?", _1.state, _1.sandbox.id] }
       if options[:history]
@@ -768,8 +787,15 @@ module LemansRemote
         end
       end
 
+      filtered = options[:running] ^ options[:complete]
+      rows.select! { (_1[1] == "running") == options[:running] } if filtered
+
       if rows.empty?
-        say "no lemans-remote runs#{options[:history] ? "" : " — try --history for completed ones"}"
+        if filtered
+          say "no #{options[:running] ? "running" : "complete"} lemans-remote runs"
+        else
+          say "no lemans-remote runs#{" — try --history for completed ones" unless options[:history]}"
+        end
         return
       end
 
@@ -783,6 +809,7 @@ module LemansRemote
     map "pull-runs" => :pull_runs
     desc "pull-runs [RUN_IDS...]", "Download archived runs from the vault into the local runs directory"
     option :all, type: :boolean, default: false, desc: "Pull every completed run, even ones already pulled"
+    option :dry_run, type: :boolean, default: false, desc: "Preview what would be downloaded without pulling anything"
     option :runs_dir, default: "runs", desc: "Local directory to sync the results into"
     def pull_runs(*run_ids)
       pulled = []
@@ -803,6 +830,16 @@ module LemansRemote
         say_status :pull, "nothing new to pull (use --all to re-pull)", :yellow if targets.empty?
 
         targets.each do |run_id|
+          if options[:dry_run]
+            size = vault.archive_size(sandbox, run_id)
+            if size
+              say_status :would_pull, "#{run_id} (#{size})", :cyan
+            else
+              say_status :missing, "#{run_id} has no runs.tar.gz (still running?)", :yellow
+            end
+            next
+          end
+
           results = vault.download_runs(sandbox, run_id, options[:runs_dir])
           if results
             vault.mark_pulled(sandbox, run_id)
@@ -828,9 +865,7 @@ module LemansRemote
 
       live = Fleet.live
       if options[:all]
-        unless live.empty?
-          raise Thor::Error, "lemans-remote: #{live.size} run(s) still live (#{live.map(&:run_id).join(", ")}) — wait for them to finish"
-        end
+        raise Thor::Error, "lemans-remote: #{live.size} run(s) still live (#{live.map(&:run_id).join(", ")}) — wait for them to finish" unless live.empty?
         return unless options[:force] || yes?("Delete the whole #{Vault::NAME} volume and every stopped run sandbox? [y/N]")
 
         Fleet.rows.each { safe_delete_sandbox(_1.sandbox) }
@@ -858,6 +893,28 @@ module LemansRemote
       raise Thor::Error, "lemans-remote: #{e.message}"
     end
 
+    desc "snapshots", "List lemans-remote orchestrator snapshots"
+    def snapshots
+      current = build_provisioner.name
+      rows = all_snapshots.select { _1.name.start_with?("#{LABEL}-") }.map do |snap|
+        shape = "#{snap.cpu.to_i}cpu/#{snap.mem.to_i}gb/#{snap.disk.to_i}gb"
+        [snap.name, snap.state, shape, snap.created_at.to_s[0, 19], snap.last_used_at.to_s[0, 19],
+         snap.name == current ? "current" : "-"]
+      end
+
+      if rows.empty?
+        say "no lemans-remote snapshots — run `lemans-remote provision` first"
+        return
+      end
+
+      rows.sort_by! { _1[3].to_s }
+      rows.reverse!
+      print_table([%w[snapshot state shape created last-used note]] + rows)
+      say "delete old ones with `lemans-remote deprovision --name SNAPSHOT`"
+    rescue Lemans::ConfigError, RuntimeError => e
+      raise Thor::Error, "lemans-remote: #{e.message}"
+    end
+
     desc "deprovision", "Delete the orchestrator snapshot"
     option :name, desc: "Delete a specific snapshot by name (default: the one for this shape and version)"
     def deprovision
@@ -877,14 +934,14 @@ module LemansRemote
       say Lemans::VERSION
     end
 
+    FANOUT_THREADS = 4
+
     private
 
     def build_provisioner
       Provisioner.new(cpus: options[:cpus].to_i, memory_gb: options[:memory].to_i, disk_gb: options[:disk].to_i)
     end
 
-    FANOUT_THREADS = 4
-
     def plan_batches(bench, tasks)
       return [tasks] if options[:sync] || options[:run_in_band]
 
@@ -892,11 +949,11 @@ module LemansRemote
       expanded.map { [_1] }
     end
 
-    def launch_batch(bench, tasks, models, provisioner)
+    def launch_batch(bench, tasks, models, provisioner, attempt: nil)
       Runner.new(
         bench: bench,
         snapshot: provisioner.name,
-        run_id: build_run_id(bench, tasks, models),
+        run_id: build_run_id(bench, tasks, models, attempt),
         tasks: tasks,
         models: models,
         extra_args: options[:args].strip,
@@ -909,24 +966,26 @@ module LemansRemote
       ).call
     end
 
-    def launch_batches(bench, batches, models, provisioner)
+    def launch_batches(bench, jobs, models, provisioner)
       queue = Queue.new
-      batches.each { queue << _1 }
+      jobs.each { queue << _1 }
       failures = Queue.new
 
-      threads = [FANOUT_THREADS, batches.size].min.times.map do
+      threads = [FANOUT_THREADS, jobs.size].min.times.map do
         Thread.new do
           loop do
-            batch =
+            batch, attempt =
               begin
                 queue.pop(true)
               rescue ThreadError
                 break
               end
             begin
-              launch_batch(bench, batch, models, provisioner)
+              launch_batch(bench, batch, models, provisioner, attempt:)
             rescue StandardError => e
-              failures << "#{batch.join(",")} (#{e.message})"
+              label = batch.empty? ? "all" : batch.join(",")
+              label += "/a#{attempt}" if attempt
+              failures << "#{label} (#{e.message})"
             end
           end
         end
@@ -936,27 +995,39 @@ module LemansRemote
       Array.new(failures.size) { failures.pop }
     end
 
-    def build_run_id(bench, tasks, models)
-      parts = [bench.root.basename.to_s]
+    def build_run_id(bench, tasks, models, attempt = nil)
+      parts = [bench.root.expand_path.basename.to_s]
       parts << (tasks.size == 1 ? tasks.first : "#{tasks.size}tasks") if tasks.any?
       if models.any?
         short = models.first.split("/").last
         short += "+#{models.size - 1}" if models.size > 1
         parts << short
       end
+      parts << "a#{attempt}" if attempt
       parts << (ENV["LEMANS_REMOTE_USER"] || ENV["USER"] || "anon")
       parts << Time.now.utc.strftime("%Y%m%dT%H%M%S")
       parts << SecureRandom.alphanumeric(4).downcase
       parts.map { name_part(_1) }.reject(&:empty?).join("-")
     end
 
-    def name_part(value) = value.to_s.downcase.gsub(/[^a-z0-9._+]+/, "-").gsub(/\A-+|-+\z/, "")
+    def name_part(value) = value.to_s.downcase.gsub(/[^a-z0-9._+]+/, "-").gsub(/\A[-.]+|[-.]+\z/, "")
+
+    def all_snapshots
+      items = []
+      page = 1
+      loop do
+        result = LemansRemote.client.snapshot.list(page: page, limit: 200)
+        items.concat(result.items)
+        break if page >= result.total_pages.to_i
+
+        page += 1
+      end
+      items
+    end
 
     def with_vault(&block)
       provisioner = build_provisioner
-      unless provisioner.provisioned?
-        raise Thor::Error, "lemans-remote: snapshot #{provisioner.name} not found — run `lemans-remote provision` first"
-      end
+      raise Thor::Error, "lemans-remote: snapshot #{provisioner.name} not found — run `lemans-remote provision` first" unless provisioner.provisioned?
 
       vault = VaultClient.new(snapshot: provisioner.name)
       vault.with_helper { |sandbox| block.call(vault, sandbox) }

data/lib/lemans/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lemans
-  VERSION = "0.2.1"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lemans
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Svyatoslav Kryukov
@@ -114,6 +114,7 @@ email:
 - ardecvz@gmail.com
 executables:
 - lemans
+- lemans-remote
 extensions: []
 extra_rdoc_files: []
 files: