sandbox-adapter 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +20 -0
- data/lib/sandbox/adapter/box.rb +195 -0
- data/lib/sandbox/adapter/checkout.rb +54 -0
- data/lib/sandbox/adapter/credentials.rb +44 -0
- data/lib/sandbox/adapter/layout.rb +51 -0
- data/lib/sandbox/adapter/pages.rb +11 -0
- data/lib/sandbox/adapter/prices.rb +21 -0
- data/lib/sandbox/adapter/provider.rb +135 -0
- data/lib/sandbox/adapter/sandbox_vm.rb +16 -0
- data/lib/sandbox/adapter/ssh.rb +112 -0
- data/lib/sandbox/adapter/steps.rb +447 -0
- data/lib/sandbox/adapter/version.rb +5 -0
- data/lib/sandbox/adapter/vm.rb +42 -0
- data/lib/sandbox/adapter.rb +45 -0
- data/lib/sandbox-adapter.rb +28 -0
- metadata +147 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
require "net/ssh"
|
|
2
|
+
require "net/sftp"
|
|
3
|
+
|
|
4
|
+
# Reaches a box over SSH: one connection per command, SFTP for files. A nil key means the user name alone is
|
|
5
|
+
# the credential (a sandbox token). host_key pins the server: nil accepts the first key seen and reports it
|
|
6
|
+
# through on_host_key; a later mismatch raises. What Hetzner and Daytona hand Sandbox::Adapter::Box.
|
|
7
|
+
class Sandbox::Adapter::Ssh
|
|
8
|
+
HostKeyMismatch = Class.new(Sandbox::Adapter::Box::Failed)
|
|
9
|
+
CONNECTION_ERRORS = [
|
|
10
|
+
Errno::ECONNREFUSED,
|
|
11
|
+
Errno::ECONNRESET,
|
|
12
|
+
Errno::EHOSTUNREACH,
|
|
13
|
+
Errno::ETIMEDOUT,
|
|
14
|
+
Errno::EPIPE,
|
|
15
|
+
Net::SSH::ConnectionTimeout,
|
|
16
|
+
Net::SSH::Disconnect,
|
|
17
|
+
SocketError,
|
|
18
|
+
IOError
|
|
19
|
+
].freeze
|
|
20
|
+
|
|
21
|
+
attr_reader :host, :port, :user, :key
|
|
22
|
+
|
|
23
|
+
def initialize(host:, user:, key:, port: 22, host_key: nil, on_host_key: nil)
|
|
24
|
+
raise Sandbox::Adapter::Box::Failed, "no address" if host.blank?
|
|
25
|
+
|
|
26
|
+
@host = host
|
|
27
|
+
@port = port
|
|
28
|
+
@user = user
|
|
29
|
+
@key = key
|
|
30
|
+
@host_key = host_key
|
|
31
|
+
@on_host_key = on_host_key
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def address = host
|
|
35
|
+
def connection_errors = CONNECTION_ERRORS
|
|
36
|
+
# A detached command's log can be followed again after a dropped link.
|
|
37
|
+
def reattaches? = true
|
|
38
|
+
|
|
39
|
+
# Runs a shell command line; each output line is yielded; returns the exit status.
|
|
40
|
+
def exec(command, &)
|
|
41
|
+
session { |link| return channel(link, command, &) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def upload(path, io) = session { |link| link.sftp.upload!(io, path) }
|
|
45
|
+
def download(path) = session { |link| return link.sftp.download!(path) }
|
|
46
|
+
|
|
47
|
+
# net-ssh options for the credential: our key, or the user name alone.
|
|
48
|
+
def auth
|
|
49
|
+
key ? { key_data: [ key ], keys: [], keys_only: true } : { auth_methods: [ "none" ], keys: [] }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def session(&)
|
|
55
|
+
Net::SSH.start(
|
|
56
|
+
host,
|
|
57
|
+
user,
|
|
58
|
+
port:,
|
|
59
|
+
**auth,
|
|
60
|
+
non_interactive: true,
|
|
61
|
+
verify_host_key: HostKeyVerifier.new(self),
|
|
62
|
+
timeout: 10,
|
|
63
|
+
keepalive: true,
|
|
64
|
+
keepalive_interval: 15,
|
|
65
|
+
&
|
|
66
|
+
)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# net-ssh hands the presented key here. Pinned key: exact match or raise. No pin: accept and report.
|
|
70
|
+
def verify_host_key(arguments)
|
|
71
|
+
key = arguments[:key]
|
|
72
|
+
presented = "#{key.ssh_type} #{Base64.strict_encode64(key.to_blob)}"
|
|
73
|
+
if @host_key
|
|
74
|
+
unless presented == @host_key
|
|
75
|
+
raise HostKeyMismatch,
|
|
76
|
+
"#{address} presented #{key.ssh_type} #{arguments[:fingerprint]}, pinned key differs"
|
|
77
|
+
end
|
|
78
|
+
else
|
|
79
|
+
@host_key = presented
|
|
80
|
+
@on_host_key&.call(presented)
|
|
81
|
+
end
|
|
82
|
+
true
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
HostKeyVerifier = Struct.new(:transport) do
|
|
86
|
+
def verify(arguments) = transport.send(:verify_host_key, arguments)
|
|
87
|
+
def verify_signature(&) = yield
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def channel(link, command)
|
|
91
|
+
status = nil
|
|
92
|
+
buffers = Hash.new { |h, k| h[k] = "".b }
|
|
93
|
+
link.open_channel do |ch|
|
|
94
|
+
ch.exec(command) do |c, _ok|
|
|
95
|
+
c.on_data do |_, data|
|
|
96
|
+
Sandbox::Adapter::Box.each_line(buffers[:out], data) { |line| yield line if block_given? }
|
|
97
|
+
end
|
|
98
|
+
c.on_extended_data do |_, _, data|
|
|
99
|
+
Sandbox::Adapter::Box.each_line(buffers[:err], data) { |line| yield line if block_given? }
|
|
100
|
+
end
|
|
101
|
+
c.on_request("exit-status") { |_, data| status = data.read_long }
|
|
102
|
+
c.on_close do
|
|
103
|
+
buffers.each_value do |rest|
|
|
104
|
+
yield rest.force_encoding("UTF-8").scrub if rest.present? && block_given?
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
link.loop
|
|
110
|
+
status
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
require "digest"
|
|
2
|
+
require "shellwords"
|
|
3
|
+
|
|
4
|
+
# The shell that makes a box ready, shared by every mode: a VM per worktree, a seed, a fork. Each step runs
|
|
5
|
+
# detached on the box under its own id and logs every line under the loggable; a non-zero exit raises.
|
|
6
|
+
class Sandbox::Adapter::Steps
|
|
7
|
+
attr_reader :layout
|
|
8
|
+
|
|
9
|
+
# mcp_script is the stub the agent talks to Rails through, handed in: a box is made the same way
|
|
10
|
+
# whichever agent runs on it, and this does not know which one wrote the script it installs. Named for
|
|
11
|
+
# the script, not the step: #mcp below is the step that installs it.
|
|
12
|
+
def initialize(
|
|
13
|
+
loggable,
|
|
14
|
+
layout:,
|
|
15
|
+
github_token:,
|
|
16
|
+
mcp_script:,
|
|
17
|
+
checkout: ->(repo) { Sandbox::Adapter::Checkout.new(repo, token: github_token) }
|
|
18
|
+
)
|
|
19
|
+
@loggable = loggable
|
|
20
|
+
@layout = layout
|
|
21
|
+
@github_token = github_token
|
|
22
|
+
# A stub that is not there writes an empty executable and the agent reaches nothing.
|
|
23
|
+
@mcp_script = mcp_script.presence or raise ArgumentError, "mcp_script is empty"
|
|
24
|
+
@checkout = checkout
|
|
25
|
+
@n = 0
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
attr_reader :mcp_script
|
|
29
|
+
|
|
30
|
+
# A package that asks a question stops the box. Two answers, because neither covers the other: the files
|
|
31
|
+
# written at the end of #users, which sudo cannot strip, and `env` in front of the apt calls that run
|
|
32
|
+
# before those files exist. Never a bare assignment after `$sudo`: a shell recognises an assignment
|
|
33
|
+
# before it expands, so with an empty $sudo it tries to run `DEBIAN_FRONTEND=noninteractive` instead.
|
|
34
|
+
#
|
|
35
|
+
# What a bare image lacks: sudo, curl, git, its own hostname; and the user everything else runs as. No-op where
|
|
36
|
+
# cloud-init did it. This is the first step to reach apt, so it is the one that waits for cloud-init: a fresh
|
|
37
|
+
# VM is still running its own installs, and two dpkg runs at once leave packages half configured.
|
|
38
|
+
#
|
|
39
|
+
# The C toolchain is the box's too: a gem, a wheel or a node module with a native extension is compiled here,
|
|
40
|
+
# whatever the runtime, so cc, make, pkg-config and the headers they link against come with the box. A library
|
|
41
|
+
# a single project needs, libpq for one, is that environment's setup command to install.
|
|
42
|
+
def users(box)
|
|
43
|
+
sh(box, <<~SH)
|
|
44
|
+
set -e
|
|
45
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
46
|
+
if [ "$(id -u)" = 0 ]; then sudo=; else sudo=sudo; fi
|
|
47
|
+
command -v cloud-init >/dev/null && { $sudo cloud-init status --wait >/dev/null || [ $? -eq 2 ]; }
|
|
48
|
+
grep -q " $(hostname)$" /etc/hosts || echo "127.0.1.1 $(hostname)" | $sudo tee -a /etc/hosts >/dev/null
|
|
49
|
+
[ -s /etc/apt/keyrings/docker.asc ] || $sudo rm -f /etc/apt/sources.list.d/docker.list /etc/apt/keyrings/docker.asc
|
|
50
|
+
for c in sudo curl git gpg cc make pkg-config; do command -v $c >/dev/null || missing=1; done
|
|
51
|
+
if [ -n "$missing" ]; then
|
|
52
|
+
$sudo apt-get update -qq
|
|
53
|
+
$sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y -qq sudo curl ca-certificates gnupg git build-essential pkg-config libssl-dev libyaml-dev zlib1g-dev
|
|
54
|
+
fi
|
|
55
|
+
id -u #{layout.user} >/dev/null 2>&1 || $sudo useradd -m -s /bin/bash -G sudo #{layout.user}
|
|
56
|
+
[ -f /etc/sudoers.d/#{layout.user} ] || echo '#{layout.user} ALL=(ALL) NOPASSWD:ALL' | $sudo tee /etc/sudoers.d/#{layout.user} >/dev/null
|
|
57
|
+
printf 'debconf debconf/frontend select Noninteractive\n' | $sudo debconf-set-selections
|
|
58
|
+
$sudo mkdir -p /etc/needrestart/conf.d
|
|
59
|
+
printf "\\$nrconf{restart} = 'a';\\n\\$nrconf{kernelhints} = 0;\\n" | $sudo tee /etc/needrestart/conf.d/nvoi.conf >/dev/null
|
|
60
|
+
$sudo mkdir -p #{layout.scratch}
|
|
61
|
+
$sudo chmod 1777 #{layout.scratch}
|
|
62
|
+
SH
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Docker installed and its daemon up: through systemd where a box has one, else started by hand.
|
|
66
|
+
def docker(box)
|
|
67
|
+
sh(box, <<~SH)
|
|
68
|
+
set -e
|
|
69
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
70
|
+
if ! command -v docker >/dev/null; then
|
|
71
|
+
sudo rm -f /usr/sbin/policy-rc.d
|
|
72
|
+
sudo install -m 0755 -d /etc/apt/keyrings
|
|
73
|
+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /tmp/docker.asc && sudo install -m 0644 /tmp/docker.asc /etc/apt/keyrings/docker.asc
|
|
74
|
+
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu #{layout.codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
|
|
75
|
+
sudo apt-get update -qq
|
|
76
|
+
sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
|
77
|
+
fi
|
|
78
|
+
sudo usermod -aG docker #{layout.user}
|
|
79
|
+
if [ -d /run/systemd/system ]; then
|
|
80
|
+
sudo systemctl enable -q --now docker
|
|
81
|
+
else
|
|
82
|
+
sudo docker info >/dev/null 2>&1 || sudo sh -c 'setsid nohup dockerd >/var/log/dockerd.log 2>&1 &'
|
|
83
|
+
fi
|
|
84
|
+
for i in $(seq 1 60); do sudo docker info >/dev/null 2>&1 && exit 0; sleep 2; done
|
|
85
|
+
tail -20 /var/log/dockerd.log 2>/dev/null; exit 1
|
|
86
|
+
SH
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# The MCP stub alone, for a box that has the tools already (this machine).
|
|
90
|
+
def mcp(box)
|
|
91
|
+
sh(box, <<~SH, env: { NVOI_MCP: mcp_script })
|
|
92
|
+
set -e
|
|
93
|
+
mkdir -p #{File.dirname(layout.mcp_bin)} #{layout.owned.join(" ")}
|
|
94
|
+
printf '%s' "$NVOI_MCP" > #{layout.mcp_bin} && chmod 755 #{layout.mcp_bin}
|
|
95
|
+
SH
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# claude, the MCP stub through which the agent asks Rails for GitHub actions, a browser for captures, and
|
|
99
|
+
# mise. No token lands here.
|
|
100
|
+
#
|
|
101
|
+
# mise is the box's, not a repository's: every language is installed the same way, and a setup command
|
|
102
|
+
# that says `mise use -g ruby@3.4.1` finds it. The profile line is for a person who logs in; a command this
|
|
103
|
+
# app runs gets the shims from Box, which names them ahead of whatever PATH the box image came with.
|
|
104
|
+
def agent(box)
|
|
105
|
+
sh(box, <<~SH, env: { NVOI_MCP: mcp_script })
|
|
106
|
+
set -e
|
|
107
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
108
|
+
command -v node >/dev/null || (curl -fsSL https://deb.nodesource.com/setup_#{layout.node_major}.x | sudo -E bash - >/dev/null && sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y -qq nodejs)
|
|
109
|
+
command -v claude >/dev/null || sudo npm install -g --silent @anthropic-ai/claude-code
|
|
110
|
+
[ -d #{layout.node_modules}/playwright-core ] || sudo npm install -g --silent playwright-core@#{layout.playwright_version}
|
|
111
|
+
printf '%s' "$NVOI_MCP" | sudo tee #{layout.mcp_bin} >/dev/null && sudo chmod 755 #{layout.mcp_bin}
|
|
112
|
+
command -v mise >/dev/null || (curl -fsSL https://mise.run | MISE_INSTALL_PATH=#{layout.mise} sudo -E sh >/dev/null)
|
|
113
|
+
printf '%s\n' 'export PATH="$HOME/.local/share/mise/shims:$PATH"' | sudo tee /etc/profile.d/mise.sh >/dev/null
|
|
114
|
+
sudo -u #{layout.user} git config --global user.name nvoi && sudo -u #{layout.user} git config --global user.email nvoi@local
|
|
115
|
+
SH
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# The data directories, on the volume when there is one, else on the box's own disk. Captures belong to the
|
|
119
|
+
# browser user, who owns nothing else.
|
|
120
|
+
def storage(box, device: nil)
|
|
121
|
+
owned = layout.owned.join(" ")
|
|
122
|
+
mount =
|
|
123
|
+
if device
|
|
124
|
+
fstab = "#{device} #{layout.mount} ext4 discard,nofail,defaults 0 0"
|
|
125
|
+
"grep -q ' #{layout.mount} ' /etc/fstab || " \
|
|
126
|
+
"echo '#{fstab}' | sudo tee -a /etc/fstab >/dev/null\n sudo mount -a"
|
|
127
|
+
else
|
|
128
|
+
"true"
|
|
129
|
+
end
|
|
130
|
+
sh(box, <<~SH)
|
|
131
|
+
set -e
|
|
132
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
133
|
+
sudo mkdir -p #{layout.mount}
|
|
134
|
+
#{mount}
|
|
135
|
+
sudo mkdir -p #{owned} && sudo chown #{layout.user}:#{layout.user} #{owned}
|
|
136
|
+
id -u #{layout.browser_user} >/dev/null 2>&1 || sudo useradd --system --home-dir #{layout.captures} --shell /usr/sbin/nologin #{layout.browser_user}
|
|
137
|
+
sudo mkdir -p #{layout.captures} && sudo chown #{layout.browser_user}:#{layout.browser_user} #{layout.captures}
|
|
138
|
+
SH
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def browser(box)
|
|
142
|
+
sh(box, <<~SH, env: { PLAYWRIGHT_BROWSERS_PATH: layout.playwright })
|
|
143
|
+
set -e
|
|
144
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
145
|
+
ls #{layout.playwright} 2>/dev/null | grep -q chromium || (sudo -E env PLAYWRIGHT_BROWSERS_PATH=#{layout.playwright} npx --yes playwright-core@#{layout.playwright_version} install --with-deps chromium && sudo apt-get install -y -qq fonts-noto-cjk && sudo chown -R #{layout.user}:#{layout.user} #{layout.playwright})
|
|
146
|
+
SH
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Where the shell server is installed on a box.
|
|
150
|
+
TERMINAL_BIN = "/usr/local/bin/nvoi-terminal".freeze
|
|
151
|
+
|
|
152
|
+
# The shell server behind the Terminal view: the binary for the box's architecture (`binaries` maps
|
|
153
|
+
# `uname -m` to a local file), uploaded only when the box's differs; the machine's key written where the
|
|
154
|
+
# server reads it on every request, so a fork is given its own without a restart; and the server up as the
|
|
155
|
+
# user, starting shells in the checkout. A unit where the box has systemd, so a wake brings it back; else
|
|
156
|
+
# started detached, its pid kept. Plain round trips, not a detached step: it is short and a retry is safe.
|
|
157
|
+
def terminal(box, binaries:, key:, port:, key_file:)
|
|
158
|
+
arch = +""
|
|
159
|
+
box.run!("uname -m") { arch << _1 }
|
|
160
|
+
arch = arch.strip
|
|
161
|
+
binary = binaries[arch] or raise ArgumentError, "no terminal server for #{arch.inspect}"
|
|
162
|
+
raise ArgumentError, "#{binary} is not built: run bin/terminal" unless File.file?(binary)
|
|
163
|
+
installed = +""
|
|
164
|
+
box.run("sha256sum #{TERMINAL_BIN} 2>/dev/null") { installed << _1 }
|
|
165
|
+
upload = "#{layout.scratch}/nvoi-terminal"
|
|
166
|
+
unless installed.start_with?(Digest::SHA256.file(binary).hexdigest)
|
|
167
|
+
File.open(binary, "rb") { box.upload(upload, _1) }
|
|
168
|
+
end
|
|
169
|
+
box.run!(terminal_script(upload, port:, key_file:),
|
|
170
|
+
env: { NVOI_TERMINAL_KEY: key, NVOI_TERMINAL_UNIT: terminal_unit(port:, key_file:) })
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# The checkout, from a bundle made here so no credential reaches the box. `branch` is checked out from origin
|
|
174
|
+
# when origin has it, else cut from `base`. A kept local branch is fast-forwarded, never overwritten;
|
|
175
|
+
# `fresh: true` (a seed, a fork) resets to what origin has.
|
|
176
|
+
def checkout(box, repo:, base:, branch:, fresh: false)
|
|
177
|
+
workspace = layout.workspace
|
|
178
|
+
bundle_path = "#{layout.scratch}/checkout.bundle"
|
|
179
|
+
@checkout.call(repo).bundle(base, branch) do |bundle|
|
|
180
|
+
File.open(bundle.path, "rb") { |io| box.upload(bundle_path, io) }
|
|
181
|
+
fetches = bundle.refs.map do |ref|
|
|
182
|
+
r = Shellwords.escape(ref)
|
|
183
|
+
"git fetch -q #{bundle_path} +#{r}:refs/remotes/origin/#{r}"
|
|
184
|
+
end
|
|
185
|
+
b = Shellwords.escape(branch)
|
|
186
|
+
origin_has_branch = bundle.refs.include?(branch)
|
|
187
|
+
place =
|
|
188
|
+
if fresh
|
|
189
|
+
"git checkout -qB #{b} origin/#{origin_has_branch ? b : Shellwords.escape(base)}"
|
|
190
|
+
else
|
|
191
|
+
<<~SH.strip
|
|
192
|
+
if git show-ref -q --verify refs/heads/#{b}; then
|
|
193
|
+
git checkout -q #{b}#{" && git merge -q --ff-only origin/#{b}" if origin_has_branch}
|
|
194
|
+
else
|
|
195
|
+
git checkout -qb #{b} origin/#{origin_has_branch ? b : Shellwords.escape(base)}
|
|
196
|
+
fi
|
|
197
|
+
SH
|
|
198
|
+
end
|
|
199
|
+
sh(box, <<~SH)
|
|
200
|
+
set -e
|
|
201
|
+
[ -d #{workspace}/.git ] || (git init -q #{workspace} && git -C #{workspace} remote add origin #{repo.github_url})
|
|
202
|
+
cd #{workspace}
|
|
203
|
+
#{fetches.join("\n")}
|
|
204
|
+
#{place}
|
|
205
|
+
rm -f #{bundle_path}
|
|
206
|
+
git rev-parse --abbrev-ref HEAD && git rev-parse HEAD
|
|
207
|
+
SH
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# The recipe's ignores into the checkout's exclude file, its setup commands, then a marker naming what they
|
|
212
|
+
# were run for: on a fork, the same marker means the seed already did it. Every command runs, a failed one
|
|
213
|
+
# included, each reported on its own; the marker is written only when all of them passed.
|
|
214
|
+
def setup(box, environment, phases: nil)
|
|
215
|
+
recipe = environment.recipe(box, layout)
|
|
216
|
+
exclude(box, recipe.ignore) if recipe.ignore.any?
|
|
217
|
+
failed = recipe.setup_commands.filter_map do |c|
|
|
218
|
+
run = -> { sh(box, c.command, env: recipe.env, dir: layout.workspace) }
|
|
219
|
+
phases ? phases.phase(:setup, name: c.name, &run) : run.call
|
|
220
|
+
nil
|
|
221
|
+
rescue StandardError => e
|
|
222
|
+
"#{c.name}: #{e.message}"
|
|
223
|
+
end
|
|
224
|
+
raise Sandbox::Adapter::Box::Failed, "setup failed: #{failed.join("; ")}" if failed.any?
|
|
225
|
+
|
|
226
|
+
digest = Shellwords.escape(recipe.setup_digest)
|
|
227
|
+
marker = "echo #{digest} $(git rev-parse HEAD) > #{layout.state}/setup.done"
|
|
228
|
+
sh(box, marker, dir: layout.workspace)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Whether the marker matches this environment at the checkout's HEAD.
|
|
232
|
+
def setup_done?(box, environment)
|
|
233
|
+
recipe = environment.recipe(box, layout)
|
|
234
|
+
marker = +""
|
|
235
|
+
command = "cat #{layout.state}/setup.done 2>/dev/null; echo; git rev-parse HEAD"
|
|
236
|
+
box.run(command, dir: layout.workspace) { |line| marker << line << "\n" }
|
|
237
|
+
done, _, head = marker.split("\n")
|
|
238
|
+
done.to_s.strip == "#{recipe.setup_digest} #{head.to_s.strip}"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# The app, (re)started on every call: one launcher per preview command, all of them up at once, then the
|
|
242
|
+
# port answering. A systemd unit each where a box has an init, with Caddy in front on the preview port;
|
|
243
|
+
# else a launcher detached in its own process group, its pid kept so the next call can replace it. Units
|
|
244
|
+
# of commands that are gone are removed first.
|
|
245
|
+
def preview(box, environment)
|
|
246
|
+
recipe = environment.recipe(box, layout)
|
|
247
|
+
commands = recipe.preview_commands.to_a
|
|
248
|
+
if commands.empty?
|
|
249
|
+
raise ArgumentError, "environment #{recipe.name} has no preview commands"
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
port = recipe.preview_port
|
|
253
|
+
env = { NVOI_ENV: env_file(recipe), NVOI_CADDY: caddyfile(port) }
|
|
254
|
+
write = []
|
|
255
|
+
start = []
|
|
256
|
+
stop = []
|
|
257
|
+
commands.each_with_index do |command, i|
|
|
258
|
+
slug = slug_of(command)
|
|
259
|
+
launcher = "#{layout.state}/preview-#{slug}"
|
|
260
|
+
unit = "nvoi-preview-#{slug}"
|
|
261
|
+
env[:"NVOI_LAUNCH_#{i}"] = launch_script(command)
|
|
262
|
+
env[:"NVOI_UNIT_#{i}"] = unit_file(launcher)
|
|
263
|
+
write << "printf '%s' \"$NVOI_LAUNCH_#{i}\" | #{layout.sudo}tee #{launcher} >/dev/null && " \
|
|
264
|
+
"#{layout.sudo}chmod 755 #{launcher}"
|
|
265
|
+
start << "printf '%s' \"$NVOI_UNIT_#{i}\" | " \
|
|
266
|
+
"sudo tee /etc/systemd/system/#{unit}.service >/dev/null"
|
|
267
|
+
stop << detached(launcher)
|
|
268
|
+
end
|
|
269
|
+
# Every unit and launcher of ours that no command claims any more, including the single one this
|
|
270
|
+
# replaced.
|
|
271
|
+
kept = commands.map { "preview-#{slug_of(_1)}" }.join(" ")
|
|
272
|
+
sh(box, <<~SH, env:)
|
|
273
|
+
set -e
|
|
274
|
+
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
|
275
|
+
#{layout.sudo}mkdir -p #{File.dirname(layout.env_file)} #{layout.state}
|
|
276
|
+
printf '%s\\n' "$NVOI_ENV" | #{layout.sudo}tee #{layout.env_file} >/dev/null && #{layout.sudo}chmod 640 #{layout.env_file} && sudo chgrp #{layout.user} #{layout.env_file}
|
|
277
|
+
#{write.join("\n")}
|
|
278
|
+
for f in #{layout.state}/preview #{layout.state}/preview-*; do
|
|
279
|
+
case " #{kept} " in *" $(basename "$f") "*) ;; *) #{layout.sudo}rm -f "$f" "$f".pid "$f".log ;; esac
|
|
280
|
+
done
|
|
281
|
+
if [ -d /run/systemd/system ]; then
|
|
282
|
+
for u in /etc/systemd/system/nvoi-preview*.service; do
|
|
283
|
+
[ -e "$u" ] || continue
|
|
284
|
+
name="$(basename "$u" .service)"
|
|
285
|
+
case " #{commands.map { "nvoi-preview-#{slug_of(_1)}" }.join(" ")} " in
|
|
286
|
+
*" $name "*) ;;
|
|
287
|
+
*) sudo systemctl disable -q --now "$name"; sudo rm -f "$u" ;;
|
|
288
|
+
esac
|
|
289
|
+
done
|
|
290
|
+
if sudo systemctl list-units --failed --plain --no-legend 'nvoi-preview*' | grep -q .; then
|
|
291
|
+
sudo systemctl reset-failed 'nvoi-preview*'
|
|
292
|
+
fi
|
|
293
|
+
#{start.join("\n ")}
|
|
294
|
+
sudo systemctl daemon-reload
|
|
295
|
+
for u in #{commands.map { "nvoi-preview-#{slug_of(_1)}" }.join(" ")}; do
|
|
296
|
+
sudo systemctl enable -q "$u" && sudo systemctl restart "$u"
|
|
297
|
+
done
|
|
298
|
+
command -v caddy >/dev/null || (sudo apt-get update -qq && sudo env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y -qq caddy)
|
|
299
|
+
printf '%s' "$NVOI_CADDY" | sudo tee /etc/caddy/Caddyfile >/dev/null
|
|
300
|
+
sudo systemctl enable -q caddy && sudo systemctl reload-or-restart caddy
|
|
301
|
+
else
|
|
302
|
+
#{stop.join("\n ")}
|
|
303
|
+
fi
|
|
304
|
+
# A port that does not answer yet is what the loop is for; one that never does falls through to the logs.
|
|
305
|
+
for i in $(seq 1 120); do
|
|
306
|
+
if (echo > /dev/tcp/127.0.0.1/#{port}) 2>/dev/null; then exit 0; fi
|
|
307
|
+
sleep 5
|
|
308
|
+
done
|
|
309
|
+
if [ -d /run/systemd/system ]; then
|
|
310
|
+
for u in #{commands.map { "nvoi-preview-#{slug_of(_1)}" }.join(" ")}; do sudo journalctl -u "$u" -n 25 --no-pager; done
|
|
311
|
+
else
|
|
312
|
+
tail -n 25 #{layout.state}/preview-*.log
|
|
313
|
+
fi
|
|
314
|
+
exit 1
|
|
315
|
+
SH
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
private
|
|
319
|
+
|
|
320
|
+
# Patterns git leaves out of status and commits, each added once.
|
|
321
|
+
def exclude(box, patterns)
|
|
322
|
+
lines = patterns.map do |p|
|
|
323
|
+
q = Shellwords.escape(p)
|
|
324
|
+
file = ".git/info/exclude"
|
|
325
|
+
"grep -qxF -- #{q} #{file} 2>/dev/null || printf '%s\\n' #{q} >> #{file}"
|
|
326
|
+
end
|
|
327
|
+
sh(box, [ "set -e", "mkdir -p .git/info", *lines ].join("\n"), dir: layout.workspace)
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Every step runs detached on the box under its own id, so a dropped link resumes instead of failing the run.
|
|
331
|
+
def sh(box, command, env: {}, dir: nil)
|
|
332
|
+
@n += 1
|
|
333
|
+
id = "#{@loggable.class.name.downcase}-#{@loggable.id}-#{@n}"
|
|
334
|
+
@loggable.log(command.strip, nil)
|
|
335
|
+
box.exec!(id, command, env:, unreachable_for: 600, dir:) do |line|
|
|
336
|
+
@loggable.log(command, line)
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def slug_of(command) = command.name.parameterize.presence || command.id
|
|
341
|
+
|
|
342
|
+
def terminal_command(port:, key_file:)
|
|
343
|
+
"#{TERMINAL_BIN} -addr :#{port} -key #{key_file} -dir #{layout.workspace}"
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# A new binary restarts the server; otherwise one already up is left with its shells. It ends by falling
|
|
347
|
+
# through, never `exit`: Box#run is a login shell, and on boat.dev an explicit exit runs .bash_logout,
|
|
348
|
+
# whose clear_console fails and turns a success into exit 1.
|
|
349
|
+
def terminal_script(upload, port:, key_file:)
|
|
350
|
+
pid = "#{layout.state}/terminal.pid"
|
|
351
|
+
detached = "cd; setsid nohup #{terminal_command(port:, key_file:)} " \
|
|
352
|
+
"> #{layout.state}/terminal.log 2>&1 & echo $! > #{pid}"
|
|
353
|
+
<<~SH
|
|
354
|
+
set -e
|
|
355
|
+
sudo mkdir -p #{File.dirname(key_file)} #{layout.state}
|
|
356
|
+
printf '%s' "$NVOI_TERMINAL_KEY" | sudo tee #{key_file} >/dev/null
|
|
357
|
+
sudo chown #{layout.user} #{key_file} && sudo chmod 600 #{key_file}
|
|
358
|
+
sudo chown #{layout.user} #{layout.state}
|
|
359
|
+
fresh=
|
|
360
|
+
if [ -f #{upload} ]; then
|
|
361
|
+
sudo install -m 755 #{upload} #{TERMINAL_BIN} && rm -f #{upload}
|
|
362
|
+
fresh=1
|
|
363
|
+
fi
|
|
364
|
+
if [ -d /run/systemd/system ]; then
|
|
365
|
+
printf '%s' "$NVOI_TERMINAL_UNIT" | sudo tee /etc/systemd/system/nvoi-terminal.service >/dev/null
|
|
366
|
+
sudo systemctl daemon-reload
|
|
367
|
+
sudo systemctl enable -q nvoi-terminal
|
|
368
|
+
if [ -n "$fresh" ]; then sudo systemctl restart nvoi-terminal; else sudo systemctl start nvoi-terminal; fi
|
|
369
|
+
elif [ -n "$fresh" ] || ! { [ -s #{pid} ] && sudo kill -0 "$(cat #{pid})" 2>/dev/null; }; then
|
|
370
|
+
if [ -s #{pid} ]; then sudo kill "$(cat #{pid})" 2>/dev/null || true; fi
|
|
371
|
+
sudo -u #{layout.user} -H sh -c #{Shellwords.escape(detached)}
|
|
372
|
+
fi
|
|
373
|
+
up=
|
|
374
|
+
for i in $(seq 1 50); do
|
|
375
|
+
if curl -fs -o /dev/null http://127.0.0.1:#{port}/health; then up=1; break; fi
|
|
376
|
+
sleep 0.2
|
|
377
|
+
done
|
|
378
|
+
[ -n "$up" ] || { echo "the terminal server does not answer on port #{port}" >&2; false; }
|
|
379
|
+
SH
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def terminal_unit(port:, key_file:)
|
|
383
|
+
<<~UNIT
|
|
384
|
+
[Unit]
|
|
385
|
+
Description=nvoi terminal
|
|
386
|
+
After=network-online.target
|
|
387
|
+
[Service]
|
|
388
|
+
User=#{layout.user}
|
|
389
|
+
ExecStart=#{terminal_command(port:, key_file:)}
|
|
390
|
+
Restart=always
|
|
391
|
+
RestartSec=1
|
|
392
|
+
[Install]
|
|
393
|
+
WantedBy=multi-user.target
|
|
394
|
+
UNIT
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
def caddyfile(port)
|
|
398
|
+
<<~CADDY
|
|
399
|
+
:#{Sandbox::Adapter::Vm::PREVIEW_PORT} {
|
|
400
|
+
reverse_proxy 127.0.0.1:#{port}
|
|
401
|
+
}
|
|
402
|
+
CADDY
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def unit_file(launcher)
|
|
406
|
+
<<~UNIT
|
|
407
|
+
[Unit]
|
|
408
|
+
After=docker.service network-online.target
|
|
409
|
+
[Service]
|
|
410
|
+
User=#{layout.user}
|
|
411
|
+
ExecStart=#{launcher}
|
|
412
|
+
Restart=always
|
|
413
|
+
RestartSec=2
|
|
414
|
+
[Install]
|
|
415
|
+
WantedBy=multi-user.target
|
|
416
|
+
UNIT
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# No exec: a command may be a pipeline or a loop, which exec cannot take. The shell stays as the
|
|
420
|
+
# parent, which systemd tracks by cgroup and a detached start kills by process group.
|
|
421
|
+
# The shims go in by hand, as they do for every other command: this script is run by systemd or by a
|
|
422
|
+
# detached start, neither of which goes through Box, and a box image may ship a runtime of its own.
|
|
423
|
+
def launch_script(command)
|
|
424
|
+
<<~LAUNCH
|
|
425
|
+
#!/bin/bash -l
|
|
426
|
+
#{"export PATH=#{layout.shims}:\"$PATH\"" if layout.shims}
|
|
427
|
+
set -a; . #{layout.env_file}; set +a
|
|
428
|
+
cd #{layout.workspace}
|
|
429
|
+
#{command.command}
|
|
430
|
+
LAUNCH
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
# The env every preview command starts with: the environment's variables and PORT.
|
|
434
|
+
def env_file(recipe)
|
|
435
|
+
recipe.env.merge(PORT: recipe.preview_port).map do |k, v|
|
|
436
|
+
%(#{k}="#{v.to_s.gsub(/["\\]/) { "\\#{_1}" }}")
|
|
437
|
+
end.join("\n")
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# Killed if it was up, then started again in its own process group, its pid kept.
|
|
441
|
+
def detached(launcher)
|
|
442
|
+
run = "setsid nohup #{launcher} > #{launcher}.log 2>&1 & echo $! > #{launcher}.pid"
|
|
443
|
+
run = "#{layout.sudo}runuser -u #{layout.user} -- sh -c '#{run}'"
|
|
444
|
+
kill = "#{layout.sudo}kill -TERM -- -$(cat #{launcher}.pid) 2>/dev/null"
|
|
445
|
+
"[ -f #{launcher}.pid ] && #{kill}; #{run}"
|
|
446
|
+
end
|
|
447
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# The traditional VM interface: a server per worktree with its own volume, booted with cloud-init, reached over
|
|
2
|
+
# SSH as the deploy user with our key. Deploy is the flow. No fork: a worktree is built from scratch on its volume.
|
|
3
|
+
class Sandbox::Adapter::Vm < Sandbox::Adapter::Provider
|
|
4
|
+
def self.type = "vm"
|
|
5
|
+
def layout = Sandbox::Adapter::Layout.for(login: "deploy", volume: true)
|
|
6
|
+
|
|
7
|
+
# Also the workspace's network and firewall, and what cloud-init boots with. The machine's data carries the
|
|
8
|
+
# address and the volume's device path.
|
|
9
|
+
def create(
|
|
10
|
+
name,
|
|
11
|
+
labels: {},
|
|
12
|
+
shape: default_shape,
|
|
13
|
+
disk: default_disk(shape),
|
|
14
|
+
workspace:,
|
|
15
|
+
user_data:
|
|
16
|
+
)
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
end
|
|
19
|
+
# The workspace's network and firewall, once its last machine is gone.
|
|
20
|
+
def destroy_network(workspace) = raise NotImplementedError
|
|
21
|
+
|
|
22
|
+
# `key` is the worktree's private key, the one its server was made with.
|
|
23
|
+
def transport(m, key:, host_key: nil, on_host_key: nil)
|
|
24
|
+
Sandbox::Adapter::Ssh.new(
|
|
25
|
+
host: m.data[:address],
|
|
26
|
+
user: layout.user,
|
|
27
|
+
key:,
|
|
28
|
+
host_key:,
|
|
29
|
+
on_host_key:
|
|
30
|
+
)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The port the preview leaves the box on: the box redirects it to the app's own port (see Steps#preview).
|
|
34
|
+
# A Worker fetches a hostname on a port Cloudflare allows, not an address on any port, so the route is the
|
|
35
|
+
# address's reverse name and this port; the firewall opens it to Cloudflare alone.
|
|
36
|
+
PREVIEW_PORT = 8080
|
|
37
|
+
|
|
38
|
+
def preview(m, _port)
|
|
39
|
+
host = m.data[:hostname].presence or return nil
|
|
40
|
+
{ host: "#{host}:#{PREVIEW_PORT}", scheme: "http", token: nil, headers: {} }
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Where boxes come from. Two interfaces on one base: Sandbox::Adapter::Vm (a server per worktree, Deploy) and
|
|
2
|
+
# Sandbox::Adapter::SandboxVm (a seed per environment and forks of it, Build and Fork). An environment picks a provider
|
|
3
|
+
# by name; the provider hands Box a transport of its own. What a customer is told about a provider, and
|
|
4
|
+
# which ones this installation offers, is Tiers.
|
|
5
|
+
module Sandbox::Adapter
|
|
6
|
+
# A machine as the provider sees it. state is starting, running, stopped or broken; data is the provider's
|
|
7
|
+
# own record of it, symbol-keyed.
|
|
8
|
+
Machine = Data.define(:id, :name, :state, :data) do
|
|
9
|
+
def running? = state == :running
|
|
10
|
+
def stopped? = state == :stopped
|
|
11
|
+
def broken? = state == :broken
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# What this module is given by the installation it runs in. Each one has a default that needs nothing,
|
|
15
|
+
# so the library stands on its own; an installation that answers better says so.
|
|
16
|
+
#
|
|
17
|
+
# All four are set from one place and, under a framework that reloads code, from a hook that runs after
|
|
18
|
+
# every reload: this module is reloadable too, and what is written here goes with it (see
|
|
19
|
+
# config/initializers/compute.rb).
|
|
20
|
+
class << self
|
|
21
|
+
attr_writer :credentials, :reporter, :price_table, :providers
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Where a failure that was handled is reported. Rails answers it; a library that is not in an app
|
|
25
|
+
# reports nowhere rather than crashing on a missing constant.
|
|
26
|
+
def self.reporter = @reporter ||= (defined?(Rails) ? Rails.error : NOWHERE)
|
|
27
|
+
|
|
28
|
+
NOWHERE = Object.new.tap { |o| o.define_singleton_method(:report) { |*, **| nil } }.freeze
|
|
29
|
+
|
|
30
|
+
# Where every provider's secrets are read from.
|
|
31
|
+
def self.credentials = @credentials ||= Sandbox::Adapter::Credentials.new
|
|
32
|
+
|
|
33
|
+
# The providers this installation has, name to class name, in the order a person is offered them. Held
|
|
34
|
+
# as strings so a name may be declared before its library is loaded, and so an installation that does
|
|
35
|
+
# not have one simply does not name it.
|
|
36
|
+
def self.providers = @providers ||= {}
|
|
37
|
+
|
|
38
|
+
# What a provider's keys cost. The vendors' published table until an installation keeps its own.
|
|
39
|
+
def self.price_table = @price_table ||= Sandbox::Adapter::Prices.new
|
|
40
|
+
|
|
41
|
+
# One shapes entry, as Sandbox::Adapter::Provider#shapes answers them.
|
|
42
|
+
def self.shape(cpu:, memory:, disk:, rate:, held: 0.0, type: nil, source:, as_of: nil)
|
|
43
|
+
{ cpu:, memory:, disk:, type:, rate: rate.to_f, held: held.to_f, source:, as_of: }
|
|
44
|
+
end
|
|
45
|
+
end
|