lemans 0.0.0.pre → 0.2.0

data/lib/lemans/clobber.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "pathname"
+
+module Lemans
+  # Deletes run directories based on the provided filters.
+  class Clobber
+    TRIAL_DIR = /\A(?<task>.+)__[A-Za-z0-9]{7}\z/
+
+    def initialize(runs_dir:, tasks: [], ttl_sec: nil, invalid: false)
+      @runs_dir = Pathname(runs_dir)
+      @tasks = Array(tasks)
+      @ttl_sec = ttl_sec
+      @invalid = invalid
+    end
+
+    def matches
+      @matches ||= select_matches
+    end
+
+    # Deletes everything it can, says what it could not, and returns what it
+    # actually removed — the caller's "deleted N" must not count survivors.
+    def call
+      deleted = matches.select do |entry|
+        FileUtils.remove_entry(entry.to_s)
+        true
+      rescue SystemCallError => e
+        warn "lemans: could not delete #{entry}: #{e.message}"
+        false
+      end
+      prune_emptied_parents(deleted)
+      deleted
+    end
+
+    private
+
+    attr_reader :runs_dir, :tasks, :ttl_sec
+
+    def select_matches
+      return [] unless runs_dir.directory?
+
+      runs_dir.glob("**/").map(&:cleanpath).sort.select do |entry|
+        task = task_name(entry)
+        next false if task.nil?
+
+        (tasks.empty? || tasks.include?(task)) &&
+          (ttl_sec.nil? || age_sec(entry) > ttl_sec) &&
+          (!@invalid || invalid?(entry))
+      end
+    end
+
+    def prune_emptied_parents(deleted)
+      root = runs_dir.cleanpath
+      deleted.each do |entry|
+        dir = entry.parent
+        while dir != root && dir.children.empty?
+          dir.rmdir
+          dir = dir.parent
+        end
+      rescue SystemCallError
+        next
+      end
+    end
+
+    def task_name(entry) = TRIAL_DIR.match(entry.basename.to_s)&.[](:task)
+
+    def age_sec(entry) = Time.now - entry.mtime
+
+    # A trial that measured nothing; an unreadable result counts — it will
+    # never be read as anything else.
+    def invalid?(entry)
+      JSON.parse(entry.join("result.json").read).dig("outcome", "scored") != true
+    rescue JSON::ParserError, SystemCallError, IOError
+      true
+    end
+  end
+end

data/lib/lemans/environments/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Lemans
+  module Environments
+    # What every backend must do, and nothing more: a narrow contract is what
+    # makes a second backend a day of work instead of a subsystem.
+    class Base
+      # Backends interleave a command's streams before we ever see them, so one
+      # output field is the honest shape.
+      ExecResult = Data.define(:command, :exit_code, :output, :duration_sec) do
+        def success? = exit_code.zero?
+      end
+
+      attr_reader :image, :resources, :network, :env, :labels, :build_timeout_sec
+
+      # `labels` is backend-agnostic trial metadata (task, trial id, phase);
+      # every backend receives it even if it has nowhere to put it.
+      def initialize(image:, resources:, network:, env: {}, labels: {}, build_timeout_sec: nil)
+        @image = image
+        @resources = resources
+        @network = network
+        @env = env
+        @labels = labels
+        @build_timeout_sec = build_timeout_sec
+      end
+
+      # Build the image and bring the sandbox up under the network policy it was
+      # constructed with; a backend that cannot honour the policy must raise.
+      def start = raise(NotImplementedError)
+
+      DEFAULT_TIMEOUT = 60
+
+      def exec(command, timeout: nil, env: {}) = raise(NotImplementedError)
+
+      def upload(local_path, remote_path) = raise(NotImplementedError)
+
+      def download(remote_path, local_path) = raise(NotImplementedError)
+
+      # Phases change what the sandbox may reach: setup pulls packages, the agent
+      # reaches the model API and nothing else, the verifier reaches nothing.
+      def network_policy=(policy)
+        raise NotImplementedError
+      end
+
+      def stop = raise(NotImplementedError)
+
+      def exec!(command, **)
+        result = exec(command, **)
+        return result if result.success?
+
+        raise InfrastructureError, "#{command} exited #{result.exit_code}: #{result.output.to_s[0, 2000]}"
+      end
+    end
+  end
+end

data/lib/lemans/environments/daytona/retries.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "daytona"
+
+module Lemans
+  module Environments
+    class Daytona
+      # Another try for calls whose repeat is free. Only reads qualify: a
+      # mutation may have landed server-side before its failure surfaced.
+      module Retries
+        # The snapshot service leaks the generated client's own error classes
+        # instead of wrapping them, so both dialects have to be caught.
+        SDK_ERRORS = [::Daytona::Sdk::Error, *::Daytona::Sdk::API_ERROR_CLASSES].freeze
+
+        READ_ATTEMPTS = 3
+        RETRY_DELAY_SEC = 2
+
+        private
+
+        def with_read_retries
+          attempts = 0
+          begin
+            yield
+          rescue *SDK_ERRORS => e
+            attempts += 1
+            raise if attempts >= READ_ATTEMPTS || !retryable?(e)
+
+            sleep RETRY_DELAY_SEC
+            retry
+          end
+        end
+
+        # Transport failures surface as status 0 (libcurl stamps refused/reset/
+        # DNS with code 0) or none, and throttling and server errors heal on
+        # their own; any other 4xx would fail the same way again.
+        def retryable?(error)
+          status = status_code(error)
+          status.nil? || status.zero? || status == 429 || status >= 500
+        end
+
+        def status_code(error)
+          return error.status_code if error.is_a?(::Daytona::Sdk::Error)
+
+          ::Daytona::Sdk.api_error_details(error)[:status_code]
+        end
+      end
+    end
+  end
+end

data/lib/lemans/environments/daytona/sdk_tweaks.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "daytona"
+require "logger"
+
+module Lemans
+  module Environments
+    class Daytona
+      # Repairs the Daytona SDK needs to be usable from a harness. Upstream ask: per-request
+      # timeouts like the Python SDK's — tracked in tmp/upstream-daytona-sdk-timeouts.md.
+      module SdkTweaks
+        GENERATED_CLIENTS = [
+          ::DaytonaApiClient, ::DaytonaToolboxApiClient, ::DaytonaAnalyticsApiClient
+        ].freeze
+
+        # Must clear the longest request: an exec long-polling server-side for
+        # SHORT_COMMAND_SEC. File transfers do not ride this cap — they go
+        # through the SDK's streaming API with their own TRANSFER_TIMEOUT.
+        HTTP_TIMEOUT_SEC = Shell::SHORT_COMMAND_SEC + 30
+
+        # The clients default to timeout=0, libcurl's "never time out"; a dropped connection then
+        # parks a thread no Thread#kill reclaims. Only that 0 is replaced; explicit config wins.
+        module Deadline
+          def timeout
+            value = super
+            value&.zero? ? HTTP_TIMEOUT_SEC : value
+          end
+        end
+
+        # `.configure` is broken — it sets a default config the SDK never reads — and `Sdk.logger`
+        # memoizes with no writer, so both are silenced by hand.
+        module Quiet
+          NULL_LOGGER = Logger.new(IO::NULL)
+
+          def logger = NULL_LOGGER
+        end
+
+        def self.apply!
+          GENERATED_CLIENTS.each { _1::Configuration.prepend(Deadline) }
+          return if ENV["DEBUG_DAYTONA"] == "1"
+
+          GENERATED_CLIENTS.each { _1::Configuration.prepend(Quiet) }
+          # The same prepend seam as everything else: poking the @logger ivar
+          # would become a silent no-op if the SDK ever renamed it.
+          ::Daytona::Sdk.singleton_class.prepend(Quiet)
+        end
+      end
+    end
+  end
+end

data/lib/lemans/environments/daytona/shell.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "daytona"
+require "securerandom"
+require "shellwords"
+
+module Lemans
+  module Environments
+    class Daytona
+      # Runs one sandbox's commands. Long commands run detached and are polled — a direct call
+      # would die at the HTTP deadline — and no async exit code exists, so the command writes its own.
+      class Shell
+        include Retries
+
+        SHORT_COMMAND_SEC = 120
+        POLL_INTERVAL_SEC = 2
+        MAX_OUTPUT_BYTES = 200_000
+        HOUSEKEEPING_TIMEOUT = 60
+
+        def initialize(sandbox)
+          @sandbox = sandbox
+          @session_id = fresh_session_id
+          sandbox.process.create_session(@session_id)
+        end
+
+        def exec(command, timeout: nil, env: {})
+          validate_env!(env)
+          started = now
+          response =
+            if timeout && timeout > SHORT_COMMAND_SEC
+              exec_in_session(command, timeout: timeout, env: env)
+            else
+              exec_directly(command, timeout: timeout, env: env)
+            end
+
+          Base::ExecResult.new(command: command, duration_sec: (now - started).round(3), **response)
+        end
+
+        private
+
+        attr_reader :sandbox
+
+        # Both transports must accept the same env: the session path serializes
+        # keys into `export` lines, where a key that is not a shell identifier
+        # would fail silently and run the command without its variable.
+        def validate_env!(env)
+          env.each_key do |key|
+            next if key.to_s.match?(/\A[A-Za-z_]\w*\z/)
+
+            raise ConfigError, "environment variable #{key.inspect} is not a shell identifier"
+          end
+        end
+
+        def exec_directly(command, timeout:, env:)
+          response = sandbox.process.exec(
+            command: command,
+            env: env.empty? ? nil : env,
+            timeout: timeout&.to_i
+          )
+          { exit_code: response.exit_code, output: response.result.to_s }
+        end
+
+        def exec_in_session(command, timeout:, env:)
+          run_id = SecureRandom.hex(6)
+          status_file = "/tmp/lemans-#{run_id}.status"
+          log_file = "/tmp/lemans-#{run_id}.log"
+          exports = env.map { |key, value| "export #{key}=#{Shellwords.escape(value.to_s)}" }.join("\n")
+
+          sandbox.process.execute_session_command(
+            session_id: @session_id,
+            req: ::Daytona::SessionExecuteRequest.new(
+              # A subshell, not a brace group: `set -e`/`exit` would take the
+              # session's own shell down and leave the status line unwritten.
+              command: "(\n#{exports}\n#{command}\n) > #{Shellwords.escape(log_file)} 2>&1\n" \
+                       "echo $? > #{Shellwords.escape(status_file)}",
+              run_async: true
+            )
+          )
+
+          exit_code = nil
+          begin
+            exit_code = await_status(status_file, timeout)
+            { exit_code: exit_code || 124, output: tail(log_file) }
+          ensure
+            # On every exit path, including a poll that died: a command that
+            # outran its budget must not keep running, and the scratch files
+            # must not stay for the model to find. Session torn down first,
+            # for the best odds the command is dead before the rm runs.
+            terminate_session! if exit_code.nil?
+            clear_scratch(log_file, status_file)
+          end
+        end
+
+        def clear_scratch(*paths)
+          command = "rm -f #{paths.map { Shellwords.escape(_1) }.join(" ")}"
+          exec_directly(command, timeout: HOUSEKEEPING_TIMEOUT, env: {})
+        rescue *SDK_ERRORS => e
+          warn "lemans: could not clear #{paths.join(", ")}: #{e.message}"
+        end
+
+        def terminate_session!
+          sandbox.process.delete_session(@session_id)
+        rescue StandardError => e
+          warn "lemans: could not delete session #{@session_id}: #{e.message}"
+        ensure
+          @session_id = fresh_session_id
+          begin
+            sandbox.process.create_session(@session_id)
+          rescue StandardError => e
+            warn "lemans: could not open a replacement session: #{e.message}"
+          end
+        end
+
+        def await_status(status_file, timeout)
+          deadline = now + timeout
+          poll = "cat #{Shellwords.escape(status_file)} 2>/dev/null"
+          loop do
+            status = with_read_retries { exec_directly(poll, timeout: HOUSEKEEPING_TIMEOUT, env: {}) }
+            value = status[:output].to_s.strip
+            return Integer(value) if value.match?(/\A\d+\z/)
+            return nil if now > deadline
+
+            sleep POLL_INTERVAL_SEC
+          end
+        end
+
+        def tail(log_file, bytes: MAX_OUTPUT_BYTES)
+          with_read_retries do
+            exec_directly("tail -c #{bytes} #{Shellwords.escape(log_file)} 2>/dev/null",
+                          timeout: HOUSEKEEPING_TIMEOUT, env: {})
+            # Scrubbed where bytes become a string: tail -c cuts on a byte
+            # boundary and can split a multibyte character.
+          end[:output].to_s.scrub
+        end
+
+        def fresh_session_id = "lemans-#{SecureRandom.hex(6)}"
+
+        def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/lemans/environments/daytona/snapshot_store.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "concurrent"
+require "daytona"
+require "digest"
+require "timeout"
+
+module Lemans
+  module Environments
+    class Daytona
+      # Content-named snapshots: one build per image digest × resource shape,
+      # reused forever after.
+      class SnapshotStore
+        include Retries
+
+        POLL_INTERVAL_SEC = 2
+
+        SNAPSHOT_FAILED = [
+          ::DaytonaApiClient::SnapshotState::ERROR,
+          ::DaytonaApiClient::SnapshotState::BUILD_FAILED
+        ].freeze
+
+        # Two trials that want the same missing snapshot must not both build
+        # it; across processes the name collision settles it and the loser waits.
+        LOCKS = Concurrent::Map.new
+
+        def self.lock(name)
+          LOCKS.compute_if_absent(name) { Mutex.new }
+        end
+
+        def initialize(client:, image:, resources:, build_timeout_sec:, logger: nil)
+          @client = client
+          @image = image
+          @resources = resources
+          @build_timeout_sec = build_timeout_sec
+          @logger = logger
+        end
+
+        def call
+          self.class.lock(name).synchronize do
+            existing = find(name)
+            existing = discard(existing) if existing && SNAPSHOT_FAILED.include?(existing.state)
+            existing ? await_ready(existing) : build(name)
+          end
+          name
+        end
+
+        # The lookup name and the whole reuse policy: image digest plus resource
+        # shape, because a sandbox inherits resources from its snapshot.
+        def name
+          @name ||= begin
+            shape = [image.digest, resources.cpus, resources.memory_mb, resources.storage_mb].join(":")
+            "lemans-#{Digest::SHA256.hexdigest(shape)[0, 32]}"
+          end
+        end
+
+        private
+
+        attr_reader :client, :image, :resources, :build_timeout_sec, :logger
+
+        # A failed build keeps the name, so it would wedge every future trial
+        # sharing that image. The state is terminal; rebuilding is all that is left.
+        def discard(snapshot)
+          client.snapshot.delete(snapshot)
+
+          deadline = now + build_timeout_sec
+          while find(name)
+            raise InfrastructureError, "daytona: failed snapshot #{name} would not go away" if now > deadline
+
+            sleep POLL_INTERVAL_SEC
+          end
+          nil
+        rescue *SDK_ERRORS => e
+          # Another process discarding the same failed snapshot got there
+          # first: gone is the goal state, same as find's 404 policy.
+          return nil if status_code(e) == 404
+
+          raise InfrastructureError, "daytona: could not remove failed snapshot #{name}: #{e.message}"
+        end
+
+        def find(name)
+          with_read_retries { client.snapshot.get(name) }
+        rescue *SDK_ERRORS => e
+          return nil if status_code(e) == 404
+
+          raise InfrastructureError, "daytona: could not look up snapshot #{name}: #{e.message}"
+        end
+
+        # The SDK's blocking create has no budget of its own, so
+        # environment.build_timeout is enforced here or nowhere.
+        def build(name)
+          params = ::Daytona::CreateSnapshotParams.new(
+            name: name,
+            # A published reference goes in as a one-line `FROM`, not a bare name:
+            # Daytona mangles a bare name's `@sha256:` pin into an invalid reference.
+            image: image.built? ? ::Daytona::Image.from_dockerfile(image.dockerfile_path.to_s) : base_image,
+            resources: daytona_resources
+          )
+          Timeout.timeout(build_timeout_sec, InfrastructureError,
+                          "daytona: snapshot #{name} did not build within #{build_timeout_sec}s") do
+            client.snapshot.create(params, on_logs: logger)
+          end
+        rescue *SDK_ERRORS => e
+          unless status_code(e) == 409
+            raise InfrastructureError,
+                  "daytona: could not build snapshot #{name}: #{e.message}"
+          end
+
+          # Another process reached the same missing snapshot first; this trial
+          # waits for its build rather than failing.
+          taken = find(name)
+          raise InfrastructureError, "daytona: snapshot #{name} was taken and then vanished" if taken.nil?
+
+          await_ready(taken)
+        end
+
+        # A snapshot that exists may still be building for whoever won the race,
+        # or deactivated from disuse — a weekly run will hit that.
+        def await_ready(snapshot)
+          deadline = now + build_timeout_sec
+          activated = false
+
+          loop do
+            state = snapshot.state
+            return if state == ::DaytonaApiClient::SnapshotState::ACTIVE
+
+            raise InfrastructureError, "daytona: snapshot #{name} is #{state}: #{snapshot.error_reason}" if SNAPSHOT_FAILED.include?(state)
+
+            if state == ::DaytonaApiClient::SnapshotState::INACTIVE && !activated
+              client.snapshot.activate(snapshot)
+              activated = true
+            end
+
+            raise InfrastructureError, "daytona: snapshot #{name} was still #{state} after #{build_timeout_sec}s" if now > deadline
+
+            sleep POLL_INTERVAL_SEC
+            snapshot = find(name)
+            if snapshot.nil?
+              raise InfrastructureError,
+                    "daytona: snapshot #{name} vanished while it was being waited on"
+            end
+          end
+        rescue *SDK_ERRORS => e
+          raise InfrastructureError, "daytona: could not wait for snapshot #{name}: #{e.message}"
+        end
+
+        def base_image = ::Daytona::Image.base(image.reference)
+
+        def daytona_resources
+          ::Daytona::Resources.new(
+            cpu: resources.cpus,
+            memory: to_gib(resources.memory_mb),
+            disk: to_gib(resources.storage_mb)
+          )
+        end
+
+        def to_gib(megabytes) = [(megabytes / 1024.0).ceil, 1].max
+
+        def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end