lemans 0.2.3 → 1.0.0.pre.1

data/lib/lemans/environment.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Lemans
+  # What every backend must do, and nothing more: a narrow contract is what
+  # makes a second backend a day of work instead of a subsystem. The
+  # name-to-class registry lives on the Environments module.
+  class Environment
+    # Backends interleave a command's streams before we ever see them, so one
+    # output field is the honest shape.
+    ExecResult = Data.define(:command, :exit_code, :output, :duration) do
+      def success? = exit_code.zero?
+    end
+
+    DEFAULT_TIMEOUT = 60
+
+    attr_reader :image, :resources, :network, :env, :labels, :build_timeout
+
+    # `labels` is backend-agnostic trial metadata (task, trial id, phase);
+    # every backend receives it even if it has nowhere to put it.
+    def initialize(image:, resources:, network:, env: {}, labels: {}, build_timeout: nil)
+      @image = image
+      @resources = resources
+      @network = network
+      @env = env
+      @labels = labels
+      @build_timeout = build_timeout
+    end
+
+    # Build the image and bring the sandbox up under the network policy it was
+    # constructed with; a backend that cannot honour the policy must raise.
+    def start = raise(NotImplementedError)
+
+    def exec(command, timeout: nil, env: {}) = raise(NotImplementedError)
+
+    def upload(local_path, remote_path) = raise(NotImplementedError)
+
+    def download(remote_path, local_path) = raise(NotImplementedError)
+
+    # Phases change what the sandbox may reach: setup pulls packages, the agent
+    # reaches the model API and nothing else, the verifier reaches nothing.
+    def switch_network_policy!(policy)
+      raise NotImplementedError
+    end
+
+    def stop = raise(NotImplementedError)
+
+    def exec!(command, **)
+      result = exec(command, **)
+      return result if result.success?
+
+      raise InfrastructureError, "#{command} exited #{result.exit_code}: #{result.output.to_s[0, 2000]}"
+    end
+  end
+end

data/lib/lemans/environments/daytona/faraday_transfer.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "daytona"
+require "faraday"
+require "faraday/multipart"
+require "json"
+
+module Lemans
+  module Environments
+    class Daytona
+      # Reroutes the SDK's streaming file transfers from typhoeus/libcurl to
+      # Faraday over Net::HTTP: pure-Ruby I/O is immune to the libcurl GC race
+      # that segfaults the VM under concurrent transfers.
+      # Only the FileTransfer entry points move — the generated JSON clients
+      # stay on typhoeus. Opt-in via .apply!
+      module FaradayTransfer
+        def self.apply!
+          ::Daytona::FileTransfer.singleton_class.prepend(Transfers)
+        end
+
+        # The prepend seam over ::Daytona::FileTransfer's entry points.
+        module Transfers
+          def stream_download(api_client:, remote_path:, timeout:, on_progress: nil, cancel_event: nil, &)
+            FaradayTransfer.download(api_client:, remote_path:, timeout:, on_progress:, cancel_event:, &)
+          end
+
+          def stream_upload(api_client:, remote_path:, source:, timeout:, on_progress: nil, cancel_event: nil)
+            FaradayTransfer.upload(api_client:, remote_path:, source:, timeout:, on_progress:, cancel_event:)
+          end
+        end
+
+        # Counts bytes as multipart-post drains the file and surfaces
+        # cancellation as an exception, which tears the connection down.
+        class UploadIO
+          def initialize(io, remote_path, on_progress:, cancel_event:)
+            @io = io
+            @remote_path = remote_path
+            @on_progress = on_progress
+            @cancel_event = cancel_event
+            @bytes_sent = 0
+          end
+
+          def read(...)
+            raise ::Daytona::Sdk::Error, "Upload cancelled: #{@remote_path}" if @cancel_event&.set?
+
+            chunk = @io.read(...)
+            if chunk && @on_progress
+              @bytes_sent += chunk.bytesize
+              @on_progress.call(::Daytona::UploadProgress.new(bytes_sent: @bytes_sent))
+            end
+            chunk
+          end
+
+          def size = @io.size
+          def length = @io.size
+          def rewind = @io.rewind
+          def eof? = @io.eof?
+          def close = @io.close
+        end
+
+        class << self
+          def download(api_client:, remote_path:, timeout:, on_progress:, cancel_event:, &block)
+            parser = nil
+            bytes_received = 0
+            sink = proc do |chunk|
+              raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
+
+              if on_progress
+                bytes_received += chunk.bytesize
+                on_progress.call(::Daytona::DownloadProgress.new(bytes_received:, total_bytes: parser&.part_total_bytes))
+              end
+              block.call(chunk)
+            end
+            parser = ::Daytona::MultipartDownloadStreamParser.new(&sink)
+
+            error_body = String.new.b
+            response = perform_download(api_client, remote_path, timeout, parser, error_body, cancel_event)
+
+            raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
+            raise ::Daytona::Sdk::Error, parser.error_message if parser.error_message
+            raise ::Daytona::Sdk::Error, "HTTP #{response.status}: #{error_body}" unless response.success?
+
+            parser.finish!
+            ::Daytona::FileTransfer.assert_download_length!(parser, remote_path)
+            nil
+          rescue Faraday::TimeoutError
+            raise ::Daytona::Sdk::Error, "Download timed out: #{remote_path}"
+          rescue Faraday::Error => e
+            raise ::Daytona::Sdk::Error, "Download failed for #{remote_path}: #{e.message}"
+          end
+
+          def upload(api_client:, remote_path:, source:, timeout:, on_progress:, cancel_event:)
+            ::Daytona::FileTransfer.with_upload_file(source, cancel_event, remote_path) do |upload_path|
+              expected_bytes = File.size(upload_path)
+              response = File.open(upload_path, "rb") do |file|
+                io = UploadIO.new(file, remote_path, on_progress:, cancel_event:)
+                perform_upload(api_client, remote_path, timeout, io)
+              end
+              raise ::Daytona::Sdk::Error, "HTTP #{response.status}: #{response.body}" unless response.success?
+
+              verify_upload(response.body, remote_path, expected_bytes)
+            end
+          rescue Faraday::TimeoutError
+            raise ::Daytona::Sdk::Error, "Upload timed out: #{remote_path}"
+          rescue Faraday::Error => e
+            raise ::Daytona::Sdk::Error, "Upload failed for #{remote_path}: #{e.message}"
+          end
+
+          private
+
+          def perform_download(api_client, remote_path, timeout, parser, error_body, cancel_event)
+            boundary_known = false
+            connection(api_client.config).post("#{api_client.config.base_url}/files/bulk-download") do |req|
+              req.headers.update(api_client.default_headers)
+              req.headers["Accept"] = "multipart/form-data"
+              req.headers["Content-Type"] = "application/json"
+              req.body = JSON.generate(paths: [remote_path])
+              apply_timeout(req, timeout)
+              req.options.on_data = proc do |chunk, _received, env|
+                raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
+
+                if (200..299).cover?(env.status)
+                  unless boundary_known
+                    ::Daytona::FileTransfer.assign_download_boundary(parser, env.response_headers["content-type"])
+                    boundary_known = true
+                  end
+                  parser << chunk
+                else
+                  error_body << chunk.b
+                end
+              end
+            end
+          end
+
+          def perform_upload(api_client, remote_path, timeout, io)
+            connection(api_client.config, multipart: true).post("#{api_client.config.base_url}/files/bulk-upload") do |req|
+              req.headers.update(api_client.default_headers)
+              # The multipart middleware supplies its own boundary-bearing type.
+              req.headers.delete("Content-Type")
+              apply_timeout(req, timeout)
+              req.body = {
+                "files[0].path" => remote_path,
+                "files[0].file" => Faraday::Multipart::FilePart.new(io, "application/octet-stream", File.basename(remote_path))
+              }
+            end
+          end
+
+          def verify_upload(body, remote_path, expected_bytes)
+            recorded = ::Daytona::FileTransfer.recorded_upload_bytes(body, remote_path)
+            return if recorded.nil? || recorded == expected_bytes
+
+            raise ::Daytona::Sdk::Error,
+                  "Upload size mismatch for #{remote_path}: sent #{expected_bytes} bytes, daemon recorded #{recorded}"
+          end
+
+          def connection(config, multipart: false)
+            Faraday.new(ssl: { verify: config.verify_ssl, verify_hostname: config.verify_ssl_host }) do |f|
+              f.request :multipart if multipart
+              f.adapter :net_http
+            end
+          end
+
+          # Typhoeus treats 0 as "no deadline"; Faraday has no such switch, so
+          # 0 falls back to Net::HTTP's per-read defaults instead.
+          def apply_timeout(req, timeout)
+            req.options.timeout = timeout if timeout&.positive?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lemans/environments/daytona/sdk_tweaks.rb

@@ -8,7 +8,7 @@ module Lemans
     class Daytona
       # Repairs the Daytona SDK needs to be usable from a harness. Upstream ask: per-request
       # timeouts like the Python SDK's — tracked in tmp/upstream-daytona-sdk-timeouts.md.
-      module SdkTweaks
+      module SDKTweaks
         GENERATED_CLIENTS = [
           ::DaytonaApiClient, ::DaytonaToolboxApiClient, ::DaytonaAnalyticsApiClient
         ].freeze

data/lib/lemans/environments/daytona/shell.rb

@@ -33,7 +33,7 @@ module Lemans
               exec_directly(command, timeout: timeout, env: env)
             end
 
-          Base::ExecResult.new(command: command, duration_sec: (now - started).round(3), **response)
+          Environment::ExecResult.new(command: command, duration: (now - started).round(3), **response)
         end
 
         private

data/lib/lemans/environments/daytona/snapshot_store.rb

@@ -28,11 +28,11 @@ module Lemans
           LOCKS.compute_if_absent(name) { Mutex.new }
         end
 
-        def initialize(client:, image:, resources:, build_timeout_sec:, logger: nil)
+        def initialize(client:, image:, resources:, build_timeout:, logger: nil)
           @client = client
           @image = image
           @resources = resources
-          @build_timeout_sec = build_timeout_sec
+          @build_timeout = build_timeout
           @logger = logger
         end
 
@@ -49,21 +49,21 @@ module Lemans
         # shape, because a sandbox inherits resources from its snapshot.
         def name
           @name ||= begin
-            shape = [image.digest, resources.cpus, resources.memory_mb, resources.storage_mb].join(":")
+            shape = [image.digest, resources.cpus, resources.memory, resources.storage].join(":")
             "lemans-#{Digest::SHA256.hexdigest(shape)[0, 32]}"
           end
         end
 
         private
 
-        attr_reader :client, :image, :resources, :build_timeout_sec, :logger
+        attr_reader :client, :image, :resources, :build_timeout, :logger
 
         # A failed build keeps the name, so it would wedge every future trial
         # sharing that image. The state is terminal; rebuilding is all that is left.
         def discard(snapshot)
           client.snapshot.delete(snapshot)
 
-          deadline = now + build_timeout_sec
+          deadline = now + build_timeout
           while find(name)
             raise InfrastructureError, "daytona: failed snapshot #{name} would not go away" if now > deadline
 
@@ -96,8 +96,8 @@ module Lemans
             image: image.built? ? ::Daytona::Image.from_dockerfile(image.dockerfile_path.to_s) : base_image,
             resources: daytona_resources
           )
-          Timeout.timeout(build_timeout_sec, InfrastructureError,
-                          "daytona: snapshot #{name} did not build within #{build_timeout_sec}s") do
+          Timeout.timeout(build_timeout, InfrastructureError,
+                          "daytona: snapshot #{name} did not build within #{build_timeout}s") do
             client.snapshot.create(params, on_logs: logger)
           end
         rescue *SDK_ERRORS => e
@@ -117,7 +117,7 @@ module Lemans
         # A snapshot that exists may still be building for whoever won the race,
         # or deactivated from disuse — a weekly run will hit that.
         def await_ready(snapshot)
-          deadline = now + build_timeout_sec
+          deadline = now + build_timeout
           activated = false
 
           loop do
@@ -131,7 +131,7 @@ module Lemans
               activated = true
             end
 
-            raise InfrastructureError, "daytona: snapshot #{name} was still #{state} after #{build_timeout_sec}s" if now > deadline
+            raise InfrastructureError, "daytona: snapshot #{name} was still #{state} after #{build_timeout}s" if now > deadline
 
             sleep POLL_INTERVAL_SEC
             snapshot = find(name)
@@ -149,8 +149,8 @@ module Lemans
         def daytona_resources
           ::Daytona::Resources.new(
             cpu: resources.cpus,
-            memory: to_gib(resources.memory_mb),
-            disk: to_gib(resources.storage_mb)
+            memory: to_gib(resources.memory),
+            disk: to_gib(resources.storage)
           )
         end
 

data/lib/lemans/environments/daytona.rb

@@ -8,22 +8,20 @@ module Lemans
   module Environments
     # Daytona sandboxes. Daytona builds images server-side into reusable content-named
     # snapshots and enforces the network policy itself.
-    class Daytona < Base
+    class Daytona < Environment
       TTL_MINUTES = 120
 
-      DEFAULT_BUILD_TIMEOUT_SEC = 600
+      DEFAULT_BUILD_TIMEOUT = 600
 
       # Workspace tarballs ride uploads/downloads, so transfers get their own
       # budget through the SDK's streaming API instead of the global HTTP cap.
       TRANSFER_TIMEOUT = 900
 
-      # File transfers ride the SDK's typhoeus/libcurl stack, which segfaults
-      # the VM under enough concurrent easy_perform calls (a GC race on string
-      # options libcurl is still copying). Transfers are seconds each, so
-      # capping them costs little; execs and lifecycle stay fully parallel.
-      TRANSFER_SLOTS = Concurrent::Semaphore.new(6)
-
-      SdkTweaks.apply!
+      SDKTweaks.apply!
+      # The SDK's typhoeus/libcurl transfers segfault the VM under concurrent
+      # easy_perform calls; Faraday/Net::HTTP is pure Ruby, so transfers need
+      # no throttling at all.
+      FaradayTransfer.apply!
 
       attr_reader :sandbox
 
@@ -43,9 +41,9 @@ module Lemans
         config
       end
 
-      def initialize(image:, resources:, network:, env: {}, labels: {}, logger: nil, build_timeout_sec: nil)
-        super(image: image, resources: resources, network: network, env: env, labels: labels,
-              build_timeout_sec: build_timeout_sec || DEFAULT_BUILD_TIMEOUT_SEC)
+      def initialize(image:, resources:, network:, env: {}, labels: {}, logger: nil, build_timeout: nil)
+        super(image:, resources:, network:, env:, labels:,
+              build_timeout: build_timeout || DEFAULT_BUILD_TIMEOUT)
         @logger = logger
       end
 
@@ -67,12 +65,10 @@ module Lemans
       end
 
       def upload(local_path, remote_path)
-        transfer do
-          # An open handle, not a path string: the SDK uploads a non-existent
-          # path AS ITS OWN BYTES, so a missing file must die here as ENOENT.
-          Pathname(local_path).open("rb") do |file|
-            sandbox.fs.upload_file_stream(file, remote_path.to_s, timeout: TRANSFER_TIMEOUT)
-          end
+        # An open handle, not a path string: the SDK uploads a non-existent
+        # path AS ITS OWN BYTES, so a missing file must die here as ENOENT.
+        Pathname(local_path).open("rb") do |file|
+          sandbox.fs.upload_file_stream(file, remote_path.to_s, timeout: TRANSFER_TIMEOUT)
         end
       rescue *Retries::SDK_ERRORS => e
         raise InfrastructureError, "daytona: could not upload #{local_path}: #{e.message}"
@@ -81,16 +77,14 @@ module Lemans
       def download(remote_path, local_path)
         local_path = Pathname(local_path)
         local_path.dirname.mkpath
-        transfer do
-          local_path.open("wb") do |file|
-            sandbox.fs.download_file_stream(remote_path.to_s, timeout: TRANSFER_TIMEOUT) { file.write(_1) }
-          end
+        local_path.open("wb") do |file|
+          sandbox.fs.download_file_stream(remote_path.to_s, timeout: TRANSFER_TIMEOUT) { file.write(_1) }
         end
       rescue *Retries::SDK_ERRORS, SystemCallError => e
         raise InfrastructureError, "daytona: could not download #{remote_path}: #{e.message}"
       end
 
-      def network_policy=(policy)
+      def switch_network_policy!(policy)
         sandbox.update_network_settings(**network_kwargs(policy, for_update: true))
         @network = policy
       rescue *Retries::SDK_ERRORS => e
@@ -120,13 +114,6 @@ module Lemans
 
       private
 
-      def transfer
-        TRANSFER_SLOTS.acquire
-        yield
-      ensure
-        TRANSFER_SLOTS.release
-      end
-
       def client = self.class.client
 
       # A sandbox inherits the snapshot's resources, so the profile's are
@@ -146,17 +133,16 @@ module Lemans
       end
 
       def snapshot_store
-        SnapshotStore.new(client: client, image: image, resources: resources,
-                          build_timeout_sec: build_timeout_sec, logger: @logger)
+        SnapshotStore.new(client:, image:, resources:, build_timeout:, logger: @logger)
       end
 
       def network_kwargs(policy, for_update: false)
         case policy.mode
-        when :none
+        when "none"
           { network_block_all: true }
-        when :public
+        when "public"
           for_update ? { network_block_all: false } : {}
-        when :allowlist
+        when "allowlist"
           raise ConfigError, "daytona: an allowlist cannot mix domains and IP targets (#{policy.hosts.join(", ")})" if policy.domains.any? && policy.ip_targets.any?
 
           {