lemans 0.2.2 → 1.0.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +30 -10
- data/exe/lemans-remote +161 -46
- data/lib/lemans/agent.rb +39 -0
- data/lib/lemans/agents/miniswen.rb +28 -25
- data/lib/lemans/agents/miniswen_installed.rb +13 -9
- data/lib/lemans/agents/nop.rb +3 -3
- data/lib/lemans/agents/oracle.rb +8 -8
- data/lib/lemans/cli/board_reporter.rb +16 -16
- data/lib/lemans/cli/progress_reporter.rb +19 -19
- data/lib/lemans/cli/report/aggregate.rb +117 -0
- data/lib/lemans/cli/report.rb +164 -0
- data/lib/lemans/cli.rb +58 -63
- data/lib/lemans/clobber.rb +17 -55
- data/lib/lemans/config/agent.rb +51 -0
- data/lib/lemans/config/conversion.rb +62 -0
- data/lib/lemans/config/environment.rb +39 -0
- data/lib/lemans/config/image_spec.rb +42 -0
- data/lib/lemans/config/network_policy.rb +55 -0
- data/lib/lemans/config/revision.rb +42 -0
- data/lib/lemans/config/setup.rb +57 -0
- data/lib/lemans/config/tree_digest.rb +26 -0
- data/lib/lemans/config/verifier.rb +72 -0
- data/lib/lemans/config.rb +107 -0
- data/lib/lemans/environment.rb +54 -0
- data/lib/lemans/environments/daytona/faraday_transfer.rb +172 -0
- data/lib/lemans/environments/daytona/sdk_tweaks.rb +1 -1
- data/lib/lemans/environments/daytona/shell.rb +1 -1
- data/lib/lemans/environments/daytona/snapshot_store.rb +11 -11
- data/lib/lemans/environments/daytona.rb +21 -35
- data/lib/lemans/result.rb +270 -0
- data/lib/lemans/runner/executor.rb +64 -0
- data/lib/lemans/runner/task.rb +62 -0
- data/lib/lemans/runner.rb +82 -0
- data/lib/lemans/store.rb +44 -0
- data/lib/lemans/stores/fs.rb +122 -0
- data/lib/lemans/task_definition.rb +194 -0
- data/lib/lemans/trial/patch.rb +76 -0
- data/lib/lemans/trial/setup.rb +66 -0
- data/lib/lemans/trial/snapshot.rb +57 -0
- data/lib/lemans/trial/verifier.rb +190 -0
- data/lib/lemans/trial.rb +113 -147
- data/lib/lemans/version.rb +1 -1
- data/lib/lemans.rb +3 -2
- data/lib/miniswen/trajectory.rb +2 -0
- metadata +58 -25
- data/lib/lemans/agents/base.rb +0 -30
- data/lib/lemans/bench.rb +0 -280
- data/lib/lemans/environments/base.rb +0 -55
- data/lib/lemans/network_policy.rb +0 -66
- data/lib/lemans/patch.rb +0 -70
- data/lib/lemans/restore_paths.rb +0 -21
- data/lib/lemans/results/aggregate.rb +0 -114
- data/lib/lemans/results/cost_source.rb +0 -13
- data/lib/lemans/results/outcome.rb +0 -36
- data/lib/lemans/results/report.rb +0 -149
- data/lib/lemans/results/sorting.rb +0 -24
- data/lib/lemans/results/tally.rb +0 -19
- data/lib/lemans/results/usage.rb +0 -24
- data/lib/lemans/run.rb +0 -152
- data/lib/lemans/setup.rb +0 -59
- data/lib/lemans/setup_files.rb +0 -36
- data/lib/lemans/snapshot.rb +0 -55
- data/lib/lemans/task.rb +0 -207
- data/lib/lemans/tree_digest.rb +0 -24
- data/lib/lemans/units.rb +0 -44
- data/lib/lemans/verifier.rb +0 -199
- /data/lib/lemans/{verifier → trial/verifier}/assets/eport-lemans.rb +0 -0
- /data/lib/lemans/{verifier → trial/verifier}/assets/lemans_minitest_reporter.rb +0 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Lemans
|
|
6
|
+
class Config
|
|
7
|
+
# A directory's contents reduced to one number. Paths hash alongside
|
|
8
|
+
# contents, and dotfiles are included — glob skips them by default.
|
|
9
|
+
module TreeDigest
|
|
10
|
+
def self.call(dir)
|
|
11
|
+
dir = Pathname(dir)
|
|
12
|
+
sha = Digest::SHA256.new
|
|
13
|
+
dir.glob("**/*", File::FNM_DOTMATCH).sort.each do |entry|
|
|
14
|
+
next unless entry.file?
|
|
15
|
+
|
|
16
|
+
path = entry.relative_path_from(dir).to_s
|
|
17
|
+
contents = entry.binread
|
|
18
|
+
sha << [path.bytesize, contents.bytesize].pack("Q>Q>")
|
|
19
|
+
sha << path
|
|
20
|
+
sha << contents
|
|
21
|
+
end
|
|
22
|
+
sha.hexdigest
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pathname"
|
|
4
|
+
|
|
5
|
+
module Lemans
|
|
6
|
+
class Config
|
|
7
|
+
class Verifier # :nodoc:
|
|
8
|
+
DEFAULT_COMMAND = "if [ -x /tests/verify ]; then exec /tests/verify; " \
|
|
9
|
+
"elif [ -f /tests/verification_test.rb ]; then exec ruby -report-lemans /tests/verification_test.rb; " \
|
|
10
|
+
"else exec bash /tests/test.sh; fi"
|
|
11
|
+
|
|
12
|
+
VERIFICATION_DIR = "verification"
|
|
13
|
+
|
|
14
|
+
class << self
|
|
15
|
+
include Conversion
|
|
16
|
+
|
|
17
|
+
def from_config(data, root: Pathname("./"))
|
|
18
|
+
return if data.nil?
|
|
19
|
+
|
|
20
|
+
conf = new
|
|
21
|
+
conf.timeout = seconds!(data["timeout"]) if data["timeout"]
|
|
22
|
+
conf.setup = Setup.from_config(data["setup"], root:) if data["setup"]
|
|
23
|
+
conf.command = data["command"] if data["command"]
|
|
24
|
+
conf.preverify = data["preverify"] if data["preverify"]
|
|
25
|
+
conf.restore_paths = restore_paths!(data["restore"]) if data["restore"]
|
|
26
|
+
conf.logs_dir = absolute_path!(data["logs_dir"]) if data["logs_dir"]
|
|
27
|
+
|
|
28
|
+
conf
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# The graded surfaces restored from the pre-agent snapshot before the
|
|
32
|
+
# command runs; a task may override the list in its frontmatter.
|
|
33
|
+
def restore_paths!(declared)
|
|
34
|
+
Array(declared).map do |path|
|
|
35
|
+
raise ConfigError, "restore path must be workdir-relative, got #{path.inspect}" if path.start_with?("/")
|
|
36
|
+
raise ConfigError, "restore path must not escape the workdir: #{path.inspect}" if path.split("/").include?("..")
|
|
37
|
+
raise ConfigError, "restore path must name something inside the workdir, got #{path.inspect}" if Pathname(path).cleanpath.to_s == "."
|
|
38
|
+
|
|
39
|
+
path
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
attr_accessor :root, :timeout, :setup, :command, :preverify, :restore_paths, :logs_dir
|
|
45
|
+
|
|
46
|
+
def initialize
|
|
47
|
+
@root = Pathname("./")
|
|
48
|
+
@timeout = 10 * 60
|
|
49
|
+
@setup = Setup.new
|
|
50
|
+
@command = DEFAULT_COMMAND
|
|
51
|
+
@preverify = nil
|
|
52
|
+
@restore_paths = []
|
|
53
|
+
@logs_dir = "/logs/verifier"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def reward_path = "#{logs_dir.chomp("/")}/reward.txt"
|
|
57
|
+
|
|
58
|
+
# [absolute, remote-relative] pairs. Shared verification files grade
|
|
59
|
+
# every trial, so they ship alongside each task's own tests.
|
|
60
|
+
def files
|
|
61
|
+
@files ||= begin
|
|
62
|
+
dir = root.join(VERIFICATION_DIR)
|
|
63
|
+
if dir.directory?
|
|
64
|
+
dir.glob("**/*", File::FNM_DOTMATCH).select(&:file?).map { [it, it.relative_path_from(dir).to_s] }
|
|
65
|
+
else
|
|
66
|
+
[]
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "pathname"
|
|
6
|
+
|
|
7
|
+
module Lemans
|
|
8
|
+
# Benchmark configuration and task definitions container.
|
|
9
|
+
class Config
|
|
10
|
+
attr_reader :root, :config_path, :tasks_dir,
|
|
11
|
+
:tasks, :version,
|
|
12
|
+
:backend, :concurrency, :attempts
|
|
13
|
+
|
|
14
|
+
# Nested configs
|
|
15
|
+
attr_reader :environment, :agent, :verifier, :setup
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
def load_file(path)
|
|
19
|
+
raise ConfigError, "bench directory not found: #{path}" unless File.directory?(path)
|
|
20
|
+
|
|
21
|
+
config_name = %w[bench.yaml bench.yml].find { File.file?(File.join(path, it)) }
|
|
22
|
+
|
|
23
|
+
raise ConfigError, "bench.yml not found" unless config_name
|
|
24
|
+
|
|
25
|
+
config_path = Pathname(File.join(path, config_name))
|
|
26
|
+
|
|
27
|
+
contents = YAML.safe_load_file(config_path.to_s, aliases: true) || {}
|
|
28
|
+
raise ConfigError, "#{path}: #{config_name} must be a mapping of sections" unless contents.is_a?(Hash)
|
|
29
|
+
|
|
30
|
+
root = Pathname(path)
|
|
31
|
+
|
|
32
|
+
sections = {}
|
|
33
|
+
sections[:version] = contents["version"]
|
|
34
|
+
sections[:tasks_dir] = contents["tasks"]
|
|
35
|
+
sections[:setup] = Setup.from_config(contents["setup"], root:)
|
|
36
|
+
sections[:agent] = Agent.from_config(contents["agent"])
|
|
37
|
+
sections[:environment] = Environment.from_config(contents["environment"])
|
|
38
|
+
sections[:verifier] = Verifier.from_config(contents["verifier"], root:)
|
|
39
|
+
sections.compact!
|
|
40
|
+
|
|
41
|
+
# Older benches declared environment-phase commands under `environment.setup`.
|
|
42
|
+
if (commands = contents.dig("environment", "setup"))
|
|
43
|
+
sections[:setup] ||= Setup.new
|
|
44
|
+
sections[:setup].commands = Array(commands) + sections[:setup].commands
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
new(root, config_path:, **sections)
|
|
48
|
+
rescue Psych::Exception => e
|
|
49
|
+
raise ConfigError, "#{path}: #{e.message}"
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def initialize(root = Pathname("./"), config_path: Pathname("./bench.yml"), version: nil, tasks_dir: "tasks",
|
|
54
|
+
setup: nil, agent: Agent.new("miniswen", "openrouter/z-ai/glm-5.2"),
|
|
55
|
+
environment: Environment.new, verifier: Verifier.new)
|
|
56
|
+
@root = root
|
|
57
|
+
@config_path = config_path
|
|
58
|
+
@version = version
|
|
59
|
+
@tasks_dir = root.join(tasks_dir)
|
|
60
|
+
@setup = setup || Setup.new
|
|
61
|
+
@agent = agent
|
|
62
|
+
@environment = environment
|
|
63
|
+
@verifier = verifier
|
|
64
|
+
@verifier.root = root
|
|
65
|
+
@concurrency = 4
|
|
66
|
+
@attempts = 1
|
|
67
|
+
@backend = "daytona"
|
|
68
|
+
@tasks = parse_tasks
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def load_options(agent: nil, model: nil, attempts: nil, concurrency: nil, backend: nil, **)
|
|
72
|
+
@agent.name = agent if agent
|
|
73
|
+
@agent.models = Array(model) if model
|
|
74
|
+
@attempts = attempts if attempts
|
|
75
|
+
@concurrency = concurrency if concurrency
|
|
76
|
+
@backend = backend if backend
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def agent_name = agent.name
|
|
80
|
+
|
|
81
|
+
def models = agent.models
|
|
82
|
+
|
|
83
|
+
# Resolved once: an hours-long run reports the bench it started from, not
|
|
84
|
+
# later tree drift.
|
|
85
|
+
def revision
|
|
86
|
+
@revision ||= Revision.detect(root)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Digest captures the state of the config and verification files. Used for resuming runs mostly
|
|
90
|
+
def digest
|
|
91
|
+
@digest ||= begin
|
|
92
|
+
sha = Digest::SHA256.new
|
|
93
|
+
sha << TreeDigest.call(root.join(Verifier::VERIFICATION_DIR))
|
|
94
|
+
sha << Digest::SHA256.file(config_path.to_s).hexdigest if config_path.file?
|
|
95
|
+
sha.hexdigest[0, 16]
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def parse_tasks
|
|
102
|
+
return [] unless tasks_dir.directory?
|
|
103
|
+
|
|
104
|
+
tasks_dir.children.select(&:directory?).sort.map { TaskDefinition.load_from_directory(self, it) }
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lemans
|
|
4
|
+
# What every backend must do, and nothing more: a narrow contract is what
|
|
5
|
+
# makes a second backend a day of work instead of a subsystem. The
|
|
6
|
+
# name-to-class registry lives on the Environments module.
|
|
7
|
+
class Environment
|
|
8
|
+
# Backends interleave a command's streams before we ever see them, so one
|
|
9
|
+
# output field is the honest shape.
|
|
10
|
+
ExecResult = Data.define(:command, :exit_code, :output, :duration) do
|
|
11
|
+
def success? = exit_code.zero?
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
DEFAULT_TIMEOUT = 60
|
|
15
|
+
|
|
16
|
+
attr_reader :image, :resources, :network, :env, :labels, :build_timeout
|
|
17
|
+
|
|
18
|
+
# `labels` is backend-agnostic trial metadata (task, trial id, phase);
|
|
19
|
+
# every backend receives it even if it has nowhere to put it.
|
|
20
|
+
def initialize(image:, resources:, network:, env: {}, labels: {}, build_timeout: nil)
|
|
21
|
+
@image = image
|
|
22
|
+
@resources = resources
|
|
23
|
+
@network = network
|
|
24
|
+
@env = env
|
|
25
|
+
@labels = labels
|
|
26
|
+
@build_timeout = build_timeout
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Build the image and bring the sandbox up under the network policy it was
|
|
30
|
+
# constructed with; a backend that cannot honour the policy must raise.
|
|
31
|
+
def start = raise(NotImplementedError)
|
|
32
|
+
|
|
33
|
+
def exec(command, timeout: nil, env: {}) = raise(NotImplementedError)
|
|
34
|
+
|
|
35
|
+
def upload(local_path, remote_path) = raise(NotImplementedError)
|
|
36
|
+
|
|
37
|
+
def download(remote_path, local_path) = raise(NotImplementedError)
|
|
38
|
+
|
|
39
|
+
# Phases change what the sandbox may reach: setup pulls packages, the agent
|
|
40
|
+
# reaches the model API and nothing else, the verifier reaches nothing.
|
|
41
|
+
def switch_network_policy!(policy)
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def stop = raise(NotImplementedError)
|
|
46
|
+
|
|
47
|
+
def exec!(command, **)
|
|
48
|
+
result = exec(command, **)
|
|
49
|
+
return result if result.success?
|
|
50
|
+
|
|
51
|
+
raise InfrastructureError, "#{command} exited #{result.exit_code}: #{result.output.to_s[0, 2000]}"
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "daytona"
|
|
4
|
+
require "faraday"
|
|
5
|
+
require "faraday/multipart"
|
|
6
|
+
require "json"
|
|
7
|
+
|
|
8
|
+
module Lemans
|
|
9
|
+
module Environments
|
|
10
|
+
class Daytona
|
|
11
|
+
# Reroutes the SDK's streaming file transfers from typhoeus/libcurl to
|
|
12
|
+
# Faraday over Net::HTTP: pure-Ruby I/O is immune to the libcurl GC race
|
|
13
|
+
# that segfaults the VM under concurrent transfers.
|
|
14
|
+
# Only the FileTransfer entry points move — the generated JSON clients
|
|
15
|
+
# stay on typhoeus. Opt-in via .apply!
|
|
16
|
+
module FaradayTransfer
|
|
17
|
+
def self.apply!
|
|
18
|
+
::Daytona::FileTransfer.singleton_class.prepend(Transfers)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The prepend seam over ::Daytona::FileTransfer's entry points.
|
|
22
|
+
module Transfers
|
|
23
|
+
def stream_download(api_client:, remote_path:, timeout:, on_progress: nil, cancel_event: nil, &)
|
|
24
|
+
FaradayTransfer.download(api_client:, remote_path:, timeout:, on_progress:, cancel_event:, &)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def stream_upload(api_client:, remote_path:, source:, timeout:, on_progress: nil, cancel_event: nil)
|
|
28
|
+
FaradayTransfer.upload(api_client:, remote_path:, source:, timeout:, on_progress:, cancel_event:)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Counts bytes as multipart-post drains the file and surfaces
|
|
33
|
+
# cancellation as an exception, which tears the connection down.
|
|
34
|
+
class UploadIO
|
|
35
|
+
def initialize(io, remote_path, on_progress:, cancel_event:)
|
|
36
|
+
@io = io
|
|
37
|
+
@remote_path = remote_path
|
|
38
|
+
@on_progress = on_progress
|
|
39
|
+
@cancel_event = cancel_event
|
|
40
|
+
@bytes_sent = 0
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def read(...)
|
|
44
|
+
raise ::Daytona::Sdk::Error, "Upload cancelled: #{@remote_path}" if @cancel_event&.set?
|
|
45
|
+
|
|
46
|
+
chunk = @io.read(...)
|
|
47
|
+
if chunk && @on_progress
|
|
48
|
+
@bytes_sent += chunk.bytesize
|
|
49
|
+
@on_progress.call(::Daytona::UploadProgress.new(bytes_sent: @bytes_sent))
|
|
50
|
+
end
|
|
51
|
+
chunk
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def size = @io.size
|
|
55
|
+
def length = @io.size
|
|
56
|
+
def rewind = @io.rewind
|
|
57
|
+
def eof? = @io.eof?
|
|
58
|
+
def close = @io.close
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
class << self
|
|
62
|
+
def download(api_client:, remote_path:, timeout:, on_progress:, cancel_event:, &block)
|
|
63
|
+
parser = nil
|
|
64
|
+
bytes_received = 0
|
|
65
|
+
sink = proc do |chunk|
|
|
66
|
+
raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
|
|
67
|
+
|
|
68
|
+
if on_progress
|
|
69
|
+
bytes_received += chunk.bytesize
|
|
70
|
+
on_progress.call(::Daytona::DownloadProgress.new(bytes_received:, total_bytes: parser&.part_total_bytes))
|
|
71
|
+
end
|
|
72
|
+
block.call(chunk)
|
|
73
|
+
end
|
|
74
|
+
parser = ::Daytona::MultipartDownloadStreamParser.new(&sink)
|
|
75
|
+
|
|
76
|
+
error_body = String.new.b
|
|
77
|
+
response = perform_download(api_client, remote_path, timeout, parser, error_body, cancel_event)
|
|
78
|
+
|
|
79
|
+
raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
|
|
80
|
+
raise ::Daytona::Sdk::Error, parser.error_message if parser.error_message
|
|
81
|
+
raise ::Daytona::Sdk::Error, "HTTP #{response.status}: #{error_body}" unless response.success?
|
|
82
|
+
|
|
83
|
+
parser.finish!
|
|
84
|
+
::Daytona::FileTransfer.assert_download_length!(parser, remote_path)
|
|
85
|
+
nil
|
|
86
|
+
rescue Faraday::TimeoutError
|
|
87
|
+
raise ::Daytona::Sdk::Error, "Download timed out: #{remote_path}"
|
|
88
|
+
rescue Faraday::Error => e
|
|
89
|
+
raise ::Daytona::Sdk::Error, "Download failed for #{remote_path}: #{e.message}"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def upload(api_client:, remote_path:, source:, timeout:, on_progress:, cancel_event:)
|
|
93
|
+
::Daytona::FileTransfer.with_upload_file(source, cancel_event, remote_path) do |upload_path|
|
|
94
|
+
expected_bytes = File.size(upload_path)
|
|
95
|
+
response = File.open(upload_path, "rb") do |file|
|
|
96
|
+
io = UploadIO.new(file, remote_path, on_progress:, cancel_event:)
|
|
97
|
+
perform_upload(api_client, remote_path, timeout, io)
|
|
98
|
+
end
|
|
99
|
+
raise ::Daytona::Sdk::Error, "HTTP #{response.status}: #{response.body}" unless response.success?
|
|
100
|
+
|
|
101
|
+
verify_upload(response.body, remote_path, expected_bytes)
|
|
102
|
+
end
|
|
103
|
+
rescue Faraday::TimeoutError
|
|
104
|
+
raise ::Daytona::Sdk::Error, "Upload timed out: #{remote_path}"
|
|
105
|
+
rescue Faraday::Error => e
|
|
106
|
+
raise ::Daytona::Sdk::Error, "Upload failed for #{remote_path}: #{e.message}"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
private
|
|
110
|
+
|
|
111
|
+
def perform_download(api_client, remote_path, timeout, parser, error_body, cancel_event)
|
|
112
|
+
boundary_known = false
|
|
113
|
+
connection(api_client.config).post("#{api_client.config.base_url}/files/bulk-download") do |req|
|
|
114
|
+
req.headers.update(api_client.default_headers)
|
|
115
|
+
req.headers["Accept"] = "multipart/form-data"
|
|
116
|
+
req.headers["Content-Type"] = "application/json"
|
|
117
|
+
req.body = JSON.generate(paths: [remote_path])
|
|
118
|
+
apply_timeout(req, timeout)
|
|
119
|
+
req.options.on_data = proc do |chunk, _received, env|
|
|
120
|
+
raise ::Daytona::Sdk::Error, "Download cancelled: #{remote_path}" if cancel_event&.set?
|
|
121
|
+
|
|
122
|
+
if (200..299).cover?(env.status)
|
|
123
|
+
unless boundary_known
|
|
124
|
+
::Daytona::FileTransfer.assign_download_boundary(parser, env.response_headers["content-type"])
|
|
125
|
+
boundary_known = true
|
|
126
|
+
end
|
|
127
|
+
parser << chunk
|
|
128
|
+
else
|
|
129
|
+
error_body << chunk.b
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def perform_upload(api_client, remote_path, timeout, io)
|
|
136
|
+
connection(api_client.config, multipart: true).post("#{api_client.config.base_url}/files/bulk-upload") do |req|
|
|
137
|
+
req.headers.update(api_client.default_headers)
|
|
138
|
+
# The multipart middleware supplies its own boundary-bearing type.
|
|
139
|
+
req.headers.delete("Content-Type")
|
|
140
|
+
apply_timeout(req, timeout)
|
|
141
|
+
req.body = {
|
|
142
|
+
"files[0].path" => remote_path,
|
|
143
|
+
"files[0].file" => Faraday::Multipart::FilePart.new(io, "application/octet-stream", File.basename(remote_path))
|
|
144
|
+
}
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def verify_upload(body, remote_path, expected_bytes)
|
|
149
|
+
recorded = ::Daytona::FileTransfer.recorded_upload_bytes(body, remote_path)
|
|
150
|
+
return if recorded.nil? || recorded == expected_bytes
|
|
151
|
+
|
|
152
|
+
raise ::Daytona::Sdk::Error,
|
|
153
|
+
"Upload size mismatch for #{remote_path}: sent #{expected_bytes} bytes, daemon recorded #{recorded}"
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def connection(config, multipart: false)
|
|
157
|
+
Faraday.new(ssl: { verify: config.verify_ssl, verify_hostname: config.verify_ssl_host }) do |f|
|
|
158
|
+
f.request :multipart if multipart
|
|
159
|
+
f.adapter :net_http
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Typhoeus treats 0 as "no deadline"; Faraday has no such switch, so
|
|
164
|
+
# 0 falls back to Net::HTTP's per-read defaults instead.
|
|
165
|
+
def apply_timeout(req, timeout)
|
|
166
|
+
req.options.timeout = timeout if timeout&.positive?
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -8,7 +8,7 @@ module Lemans
|
|
|
8
8
|
class Daytona
|
|
9
9
|
# Repairs the Daytona SDK needs to be usable from a harness. Upstream ask: per-request
|
|
10
10
|
# timeouts like the Python SDK's — tracked in tmp/upstream-daytona-sdk-timeouts.md.
|
|
11
|
-
module
|
|
11
|
+
module SDKTweaks
|
|
12
12
|
GENERATED_CLIENTS = [
|
|
13
13
|
::DaytonaApiClient, ::DaytonaToolboxApiClient, ::DaytonaAnalyticsApiClient
|
|
14
14
|
].freeze
|
|
@@ -33,7 +33,7 @@ module Lemans
|
|
|
33
33
|
exec_directly(command, timeout: timeout, env: env)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
Environment::ExecResult.new(command: command, duration: (now - started).round(3), **response)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
private
|
|
@@ -28,11 +28,11 @@ module Lemans
|
|
|
28
28
|
LOCKS.compute_if_absent(name) { Mutex.new }
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
def initialize(client:, image:, resources:,
|
|
31
|
+
def initialize(client:, image:, resources:, build_timeout:, logger: nil)
|
|
32
32
|
@client = client
|
|
33
33
|
@image = image
|
|
34
34
|
@resources = resources
|
|
35
|
-
@
|
|
35
|
+
@build_timeout = build_timeout
|
|
36
36
|
@logger = logger
|
|
37
37
|
end
|
|
38
38
|
|
|
@@ -49,21 +49,21 @@ module Lemans
|
|
|
49
49
|
# shape, because a sandbox inherits resources from its snapshot.
|
|
50
50
|
def name
|
|
51
51
|
@name ||= begin
|
|
52
|
-
shape = [image.digest, resources.cpus, resources.
|
|
52
|
+
shape = [image.digest, resources.cpus, resources.memory, resources.storage].join(":")
|
|
53
53
|
"lemans-#{Digest::SHA256.hexdigest(shape)[0, 32]}"
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
private
|
|
58
58
|
|
|
59
|
-
attr_reader :client, :image, :resources, :
|
|
59
|
+
attr_reader :client, :image, :resources, :build_timeout, :logger
|
|
60
60
|
|
|
61
61
|
# A failed build keeps the name, so it would wedge every future trial
|
|
62
62
|
# sharing that image. The state is terminal; rebuilding is all that is left.
|
|
63
63
|
def discard(snapshot)
|
|
64
64
|
client.snapshot.delete(snapshot)
|
|
65
65
|
|
|
66
|
-
deadline = now +
|
|
66
|
+
deadline = now + build_timeout
|
|
67
67
|
while find(name)
|
|
68
68
|
raise InfrastructureError, "daytona: failed snapshot #{name} would not go away" if now > deadline
|
|
69
69
|
|
|
@@ -96,8 +96,8 @@ module Lemans
|
|
|
96
96
|
image: image.built? ? ::Daytona::Image.from_dockerfile(image.dockerfile_path.to_s) : base_image,
|
|
97
97
|
resources: daytona_resources
|
|
98
98
|
)
|
|
99
|
-
Timeout.timeout(
|
|
100
|
-
"daytona: snapshot #{name} did not build within #{
|
|
99
|
+
Timeout.timeout(build_timeout, InfrastructureError,
|
|
100
|
+
"daytona: snapshot #{name} did not build within #{build_timeout}s") do
|
|
101
101
|
client.snapshot.create(params, on_logs: logger)
|
|
102
102
|
end
|
|
103
103
|
rescue *SDK_ERRORS => e
|
|
@@ -117,7 +117,7 @@ module Lemans
|
|
|
117
117
|
# A snapshot that exists may still be building for whoever won the race,
|
|
118
118
|
# or deactivated from disuse — a weekly run will hit that.
|
|
119
119
|
def await_ready(snapshot)
|
|
120
|
-
deadline = now +
|
|
120
|
+
deadline = now + build_timeout
|
|
121
121
|
activated = false
|
|
122
122
|
|
|
123
123
|
loop do
|
|
@@ -131,7 +131,7 @@ module Lemans
|
|
|
131
131
|
activated = true
|
|
132
132
|
end
|
|
133
133
|
|
|
134
|
-
raise InfrastructureError, "daytona: snapshot #{name} was still #{state} after #{
|
|
134
|
+
raise InfrastructureError, "daytona: snapshot #{name} was still #{state} after #{build_timeout}s" if now > deadline
|
|
135
135
|
|
|
136
136
|
sleep POLL_INTERVAL_SEC
|
|
137
137
|
snapshot = find(name)
|
|
@@ -149,8 +149,8 @@ module Lemans
|
|
|
149
149
|
def daytona_resources
|
|
150
150
|
::Daytona::Resources.new(
|
|
151
151
|
cpu: resources.cpus,
|
|
152
|
-
memory: to_gib(resources.
|
|
153
|
-
disk: to_gib(resources.
|
|
152
|
+
memory: to_gib(resources.memory),
|
|
153
|
+
disk: to_gib(resources.storage)
|
|
154
154
|
)
|
|
155
155
|
end
|
|
156
156
|
|
|
@@ -8,22 +8,20 @@ module Lemans
|
|
|
8
8
|
module Environments
|
|
9
9
|
# Daytona sandboxes. Daytona builds images server-side into reusable content-named
|
|
10
10
|
# snapshots and enforces the network policy itself.
|
|
11
|
-
class Daytona <
|
|
11
|
+
class Daytona < Environment
|
|
12
12
|
TTL_MINUTES = 120
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
DEFAULT_BUILD_TIMEOUT = 600
|
|
15
15
|
|
|
16
16
|
# Workspace tarballs ride uploads/downloads, so transfers get their own
|
|
17
17
|
# budget through the SDK's streaming API instead of the global HTTP cap.
|
|
18
18
|
TRANSFER_TIMEOUT = 900
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
SdkTweaks.apply!
|
|
20
|
+
SDKTweaks.apply!
|
|
21
|
+
# The SDK's typhoeus/libcurl transfers segfault the VM under concurrent
|
|
22
|
+
# easy_perform calls; Faraday/Net::HTTP is pure Ruby, so transfers need
|
|
23
|
+
# no throttling at all.
|
|
24
|
+
FaradayTransfer.apply!
|
|
27
25
|
|
|
28
26
|
attr_reader :sandbox
|
|
29
27
|
|
|
@@ -43,9 +41,9 @@ module Lemans
|
|
|
43
41
|
config
|
|
44
42
|
end
|
|
45
43
|
|
|
46
|
-
def initialize(image:, resources:, network:, env: {}, labels: {}, logger: nil,
|
|
47
|
-
super(image
|
|
48
|
-
|
|
44
|
+
def initialize(image:, resources:, network:, env: {}, labels: {}, logger: nil, build_timeout: nil)
|
|
45
|
+
super(image:, resources:, network:, env:, labels:,
|
|
46
|
+
build_timeout: build_timeout || DEFAULT_BUILD_TIMEOUT)
|
|
49
47
|
@logger = logger
|
|
50
48
|
end
|
|
51
49
|
|
|
@@ -67,12 +65,10 @@ module Lemans
|
|
|
67
65
|
end
|
|
68
66
|
|
|
69
67
|
def upload(local_path, remote_path)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
sandbox.fs.upload_file_stream(file, remote_path.to_s, timeout: TRANSFER_TIMEOUT)
|
|
75
|
-
end
|
|
68
|
+
# An open handle, not a path string: the SDK uploads a non-existent
|
|
69
|
+
# path AS ITS OWN BYTES, so a missing file must die here as ENOENT.
|
|
70
|
+
Pathname(local_path).open("rb") do |file|
|
|
71
|
+
sandbox.fs.upload_file_stream(file, remote_path.to_s, timeout: TRANSFER_TIMEOUT)
|
|
76
72
|
end
|
|
77
73
|
rescue *Retries::SDK_ERRORS => e
|
|
78
74
|
raise InfrastructureError, "daytona: could not upload #{local_path}: #{e.message}"
|
|
@@ -81,16 +77,14 @@ module Lemans
|
|
|
81
77
|
def download(remote_path, local_path)
|
|
82
78
|
local_path = Pathname(local_path)
|
|
83
79
|
local_path.dirname.mkpath
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
sandbox.fs.download_file_stream(remote_path.to_s, timeout: TRANSFER_TIMEOUT) { file.write(_1) }
|
|
87
|
-
end
|
|
80
|
+
local_path.open("wb") do |file|
|
|
81
|
+
sandbox.fs.download_file_stream(remote_path.to_s, timeout: TRANSFER_TIMEOUT) { file.write(_1) }
|
|
88
82
|
end
|
|
89
83
|
rescue *Retries::SDK_ERRORS, SystemCallError => e
|
|
90
84
|
raise InfrastructureError, "daytona: could not download #{remote_path}: #{e.message}"
|
|
91
85
|
end
|
|
92
86
|
|
|
93
|
-
def
|
|
87
|
+
def switch_network_policy!(policy)
|
|
94
88
|
sandbox.update_network_settings(**network_kwargs(policy, for_update: true))
|
|
95
89
|
@network = policy
|
|
96
90
|
rescue *Retries::SDK_ERRORS => e
|
|
@@ -120,13 +114,6 @@ module Lemans
|
|
|
120
114
|
|
|
121
115
|
private
|
|
122
116
|
|
|
123
|
-
def transfer
|
|
124
|
-
TRANSFER_SLOTS.acquire
|
|
125
|
-
yield
|
|
126
|
-
ensure
|
|
127
|
-
TRANSFER_SLOTS.release
|
|
128
|
-
end
|
|
129
|
-
|
|
130
117
|
def client = self.class.client
|
|
131
118
|
|
|
132
119
|
# A sandbox inherits the snapshot's resources, so the profile's are
|
|
@@ -146,17 +133,16 @@ module Lemans
|
|
|
146
133
|
end
|
|
147
134
|
|
|
148
135
|
def snapshot_store
|
|
149
|
-
SnapshotStore.new(client
|
|
150
|
-
build_timeout_sec: build_timeout_sec, logger: @logger)
|
|
136
|
+
SnapshotStore.new(client:, image:, resources:, build_timeout:, logger: @logger)
|
|
151
137
|
end
|
|
152
138
|
|
|
153
139
|
def network_kwargs(policy, for_update: false)
|
|
154
140
|
case policy.mode
|
|
155
|
-
when
|
|
141
|
+
when "none"
|
|
156
142
|
{ network_block_all: true }
|
|
157
|
-
when
|
|
143
|
+
when "public"
|
|
158
144
|
for_update ? { network_block_all: false } : {}
|
|
159
|
-
when
|
|
145
|
+
when "allowlist"
|
|
160
146
|
raise ConfigError, "daytona: an allowlist cannot mix domains and IP targets (#{policy.hosts.join(", ")})" if policy.domains.any? && policy.ip_targets.any?
|
|
161
147
|
|
|
162
148
|
{
|