@exla/rudder 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cloud/README.md +337 -0
- package/cloud/host/90-rudder.conf +10 -0
- package/cloud/host/rudder-home-apply-update +66 -0
- package/cloud/host/rudder-home-start +37 -0
- package/cloud/host/rudder-home-update.path +9 -0
- package/cloud/host/rudder-home-update.service +9 -0
- package/cloud/host/rudder-home.service +17 -0
- package/cloud/host/rudder-host-setup +65 -0
- package/cloud/package.json +37 -0
- package/dist/cloud-host.d.ts +26 -0
- package/dist/cloud-host.js +229 -0
- package/dist/cloud-host.js.map +1 -0
- package/dist/cloud.js +11 -0
- package/dist/cloud.js.map +1 -1
- package/dist/home-scratch.d.ts +1 -0
- package/dist/home-scratch.js +23 -0
- package/dist/home-scratch.js.map +1 -0
- package/dist/jj.js +2 -1
- package/dist/jj.js.map +1 -1
- package/dist/native/darwin-arm64/native-manifest.json +3 -3
- package/dist/native/darwin-arm64/rudder-native +0 -0
- package/dist/native/linux-x64/native-manifest.json +3 -3
- package/dist/native/linux-x64/rudder-native +0 -0
- package/dist/run-manager.js +2 -1
- package/dist/run-manager.js.map +1 -1
- package/dist/warm-snapshot.d.ts +2 -1
- package/dist/warm-snapshot.js +14 -2
- package/dist/warm-snapshot.js.map +1 -1
- package/package.json +2 -1
package/cloud/README.md
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
# Rudder Cloud
|
|
2
|
+
|
|
3
|
+
Rudder Cloud is the control plane for `/login`, `/cloud`, and `/sail`.
|
|
4
|
+
|
|
5
|
+
The CLI package stays light. This service is a separate deployable app that uses
|
|
6
|
+
Better Auth for Google and GitHub login, then issues a Rudder CLI token for the
|
|
7
|
+
local dashboard.
|
|
8
|
+
|
|
9
|
+
## Quickstart (copy/paste)
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install -g @exla/rudder@latest
|
|
13
|
+
rudder login # opens the cloud login
|
|
14
|
+
rudder cloud "fix the failing tests" # launches a fresh cloud instance
|
|
15
|
+
rudder cloud list # see every instance on your account
|
|
16
|
+
rudder cloud talk <id> "what are you doing?" # message any instance, see its reply
|
|
17
|
+
rudder cloud output <id> # latest output from an instance
|
|
18
|
+
rudder cloud attach <id> # full interactive terminal
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
`rudder cloud quickstart` prints this list at any time.
|
|
22
|
+
|
|
23
|
+
## Talk to instances (HTTP relay)
|
|
24
|
+
|
|
25
|
+
Every running instance keeps a worker WebSocket open to the control plane. Two
|
|
26
|
+
endpoints let you converse with it without a full interactive attach — this is
|
|
27
|
+
what `rudder cloud talk` and the Slack bot use:
|
|
28
|
+
|
|
29
|
+
```text
|
|
30
|
+
POST /api/rudder/sail/:id/input { "text": "...", "submit": true }
|
|
31
|
+
GET /api/rudder/sail/:id/output -> { id, connected, output }
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Both require the account's CLI bearer token and instance ownership. `input`
|
|
35
|
+
injects the text into the agent's PTY exactly as a keystroke (with a trailing
|
|
36
|
+
carriage return when `submit` is not `false`); `output` returns the recent
|
|
37
|
+
replay buffer.
|
|
38
|
+
|
|
39
|
+
## Slack — the shared "main panel"
|
|
40
|
+
|
|
41
|
+
A single Slack channel becomes the control surface for every cloud instance on
|
|
42
|
+
the account. Each instance opens its own **thread** in the channel: launching
|
|
43
|
+
announces it, the agent's final result posts into the thread, and **replying in
|
|
44
|
+
a thread sends your message straight to that instance**. You can also drive it
|
|
45
|
+
with `list`, `talk <id> <message>`, `output <id>`, and `stop <id>`.
|
|
46
|
+
|
|
47
|
+
Default channel: `C0B78TDLM5G` (override with `RUDDER_SLACK_CHANNEL`).
|
|
48
|
+
|
|
49
|
+
Set up in three steps (or run `rudder cloud slack` for the guided version):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# 1. Create a Slack app from the manifest:
|
|
53
|
+
rudder cloud slack manifest # prints a JSON manifest to paste at api.slack.com/apps
|
|
54
|
+
|
|
55
|
+
# 2. Install it, then invite the bot to the channel:
|
|
56
|
+
# /invite @rudder (in C0B78TDLM5G)
|
|
57
|
+
|
|
58
|
+
# 3. Give the control plane the credentials and redeploy:
|
|
59
|
+
flyctl secrets set \
|
|
60
|
+
SLACK_BOT_TOKEN=xoxb-... \
|
|
61
|
+
SLACK_SIGNING_SECRET=... \
|
|
62
|
+
RUDDER_SLACK_CHANNEL=C0B78TDLM5G \
|
|
63
|
+
-a rudder-cloud-control
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The events request URL is `https://<control-plane>/api/slack/events`. Inbound
|
|
67
|
+
requests are verified with `SLACK_SIGNING_SECRET` (v0 signature, 5-minute replay
|
|
68
|
+
window) and deduped by Slack `event_id`.
|
|
69
|
+
|
|
70
|
+
## Environment
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
BETTER_AUTH_URL=https://cloud.example.com
|
|
74
|
+
BETTER_AUTH_SECRET=<random secret>
|
|
75
|
+
GOOGLE_CLIENT_ID=<google oauth client id>
|
|
76
|
+
GOOGLE_CLIENT_SECRET=<google oauth client secret>
|
|
77
|
+
GITHUB_CLIENT_ID=<github oauth client id>
|
|
78
|
+
GITHUB_CLIENT_SECRET=<github oauth client secret>
|
|
79
|
+
RUDDER_CLOUD_DATA_DIR=/data
|
|
80
|
+
RUDDER_S3_BUCKET=<snapshot bucket>
|
|
81
|
+
RUDDER_CLOUD_STATE_KEY=control-plane/rudder-cloud.sqlite
|
|
82
|
+
RUDDER_CLOUD_PERSIST_STATE=1
|
|
83
|
+
AWS_REGION=us-east-1
|
|
84
|
+
FLY_API_TOKEN=<fly token>
|
|
85
|
+
FLY_APP_NAME=<existing fly machines app>
|
|
86
|
+
FLY_REGION=iad
|
|
87
|
+
RUDDER_WORKER_IMAGE=<registry image for cloud/worker/Dockerfile>
|
|
88
|
+
RUDDER_SECRETS_KEY=<32 random bytes, base64 — enables the secrets vault>
|
|
89
|
+
SLACK_BOT_TOKEN=<xoxb- bot token, enables the Slack control surface>
|
|
90
|
+
SLACK_SIGNING_SECRET=<verifies inbound Slack events>
|
|
91
|
+
RUDDER_SLACK_CHANNEL=C0B78TDLM5G
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
`SLACK_*` are optional; when `SLACK_BOT_TOKEN` is unset the Slack surface is
|
|
95
|
+
simply disabled. See "Slack — the shared main panel" below.
|
|
96
|
+
|
|
97
|
+
`FLY_API_TOKEN` and `FLY_APP_NAME` are only required for the managed Fly
|
|
98
|
+
Machines runtime. BYOC runs still require `RUDDER_S3_BUCKET` and
|
|
99
|
+
`RUDDER_WORKER_IMAGE` so the control plane can store a snapshot and print a
|
|
100
|
+
worker command for the user's server.
|
|
101
|
+
|
|
102
|
+
`RUDDER_SECRETS_KEY` is the KEK for the per-account secrets vault
|
|
103
|
+
(`rudder cloud secrets ...`). Generate with `openssl rand -base64 32` and set
|
|
104
|
+
it via `flyctl secrets set`. **Back it up out-of-band**: losing it makes all
|
|
105
|
+
stored vault ciphertext unreadable (recovery = re-run `rudder cloud secrets
|
|
106
|
+
sync` from a laptop that still has the credentials). When unset, the vault
|
|
107
|
+
endpoints return 503 and clients fall back to carrying credentials inside
|
|
108
|
+
workspace snapshots (the legacy behavior, also forceable with
|
|
109
|
+
`RUDDER_CLOUD_LEGACY_SNAPSHOT_SECRETS=1` on the client).
|
|
110
|
+
|
|
111
|
+
Current hosted control plane:
|
|
112
|
+
|
|
113
|
+
```text
|
|
114
|
+
https://rudder-cloud-control.fly.dev
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Current Exla defaults:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
RUDDER_S3_BUCKET=rudder-cloud-snapshots-597088032164-us-east-1
|
|
121
|
+
AWS_REGION=us-east-1
|
|
122
|
+
FLY_APP_NAME=rudder-workers-exla
|
|
123
|
+
FLY_REGION=iad
|
|
124
|
+
RUDDER_WORKER_IMAGE=public.ecr.aws/exla/rudder-worker:latest
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The current control-plane image is:
|
|
128
|
+
|
|
129
|
+
```text
|
|
130
|
+
public.ecr.aws/exla/rudder-cloud-control:latest
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Generated AWS secrets:
|
|
134
|
+
|
|
135
|
+
- `rudder/better-auth-secret`
|
|
136
|
+
- `rudder/fly-api-token`
|
|
137
|
+
|
|
138
|
+
Google/GitHub OAuth client IDs and client secrets still need to be created in
|
|
139
|
+
the provider consoles and added as Fly secrets before the hosted login flow
|
|
140
|
+
can go live.
|
|
141
|
+
|
|
142
|
+
Until provider OAuth clients are installed, login still works through an
|
|
143
|
+
already-authenticated GitHub CLI or GitHub's device flow. `rudder login` sends
|
|
144
|
+
the resulting GitHub token to `/api/cli/login/github-token`; the server verifies
|
|
145
|
+
it with GitHub and stores only a hashed Rudder Cloud token. The hosted
|
|
146
|
+
`/cli/login` page also exposes a GitHub device-login path so browser login does
|
|
147
|
+
not dead-end while Google/GitHub Better Auth provider secrets are missing.
|
|
148
|
+
|
|
149
|
+
GitHub browser OAuth can be configured from the hosted setup page without
|
|
150
|
+
copying secrets by hand:
|
|
151
|
+
|
|
152
|
+
```text
|
|
153
|
+
https://rudder-cloud-control.fly.dev/setup/github
|
|
154
|
+
https://rudder-cloud-control.fly.dev/setup/github?org=exla
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
That page uses GitHub's App Manifest flow to create a GitHub App, receives the
|
|
158
|
+
generated OAuth client ID and secret at `/setup/github/callback`, stores them in
|
|
159
|
+
Rudder Cloud's persisted state, and rebuilds Better Auth dynamically. The normal
|
|
160
|
+
GitHub browser login button appears on `/cli/login` immediately after setup.
|
|
161
|
+
|
|
162
|
+
If the GitHub App already exists, generate a fresh client secret from the app
|
|
163
|
+
settings page and install the existing app credentials from a logged-in admin
|
|
164
|
+
CLI:
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
rudder cloud login
|
|
168
|
+
rudder cloud setup-github <client-id>
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
The CLI prompts for the client secret without echoing it. For scripts, set
|
|
172
|
+
`RUDDER_GITHUB_CLIENT_ID` and `RUDDER_GITHUB_CLIENT_SECRET`.
|
|
173
|
+
|
|
174
|
+
Google browser OAuth can be installed the same way after creating an OAuth web
|
|
175
|
+
client with this redirect URI:
|
|
176
|
+
|
|
177
|
+
```text
|
|
178
|
+
https://rudder-cloud-control.fly.dev/api/auth/callback/google
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
rudder cloud login
|
|
183
|
+
rudder cloud setup-google <client-id>
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
The CLI prompts for the Google client secret without echoing it. For scripts,
|
|
187
|
+
set `RUDDER_GOOGLE_CLIENT_ID` and `RUDDER_GOOGLE_CLIENT_SECRET`.
|
|
188
|
+
|
|
189
|
+
The setup endpoint is restricted to `RUDDER_ADMIN_EMAILS` (defaults to
|
|
190
|
+
`viraat.laldas@gmail.com,viraat@exla.ai`) and persists the credentials to S3
|
|
191
|
+
before returning.
|
|
192
|
+
|
|
193
|
+
OAuth callback URLs:
|
|
194
|
+
|
|
195
|
+
```text
|
|
196
|
+
https://cloud.example.com/api/auth/callback/google
|
|
197
|
+
https://cloud.example.com/api/auth/callback/github
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
## Local
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
npm install
|
|
204
|
+
npm run build
|
|
205
|
+
npm start
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Then point the CLI at it:
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
export RUDDER_CLOUD_URL=http://localhost:3000
|
|
212
|
+
rudder login
|
|
213
|
+
rudder cloud list
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## AWS
|
|
217
|
+
|
|
218
|
+
The AWS role of the control plane is S3 snapshot storage. The service stores
|
|
219
|
+
uploaded launch/onload snapshots in `RUDDER_S3_BUCKET` using server-side
|
|
220
|
+
encryption and gives each Fly Machine a one-hour presigned URL. Fly workers do
|
|
221
|
+
not receive AWS credentials.
|
|
222
|
+
|
|
223
|
+
The control plane also persists its SQLite state to S3 at
|
|
224
|
+
`RUDDER_CLOUD_STATE_KEY` by default. That keeps CLI tokens, sail records, worker
|
|
225
|
+
heartbeats, and Better Auth tables available across control-plane restarts
|
|
226
|
+
without requiring a database server. Set `RUDDER_CLOUD_PERSIST_STATE=0` to
|
|
227
|
+
disable that behavior for local development.
|
|
228
|
+
|
|
229
|
+
## Hosting
|
|
230
|
+
|
|
231
|
+
The control plane runs on a single Fly Machine in the `rudder-cloud-control`
|
|
232
|
+
app. WebSocket Upgrade is required for `cloud attach`, so the host must not
|
|
233
|
+
strip Upgrade headers. The Fly Machines API is used for both this app and the
|
|
234
|
+
worker app, but they are separate apps — the control plane reads
|
|
235
|
+
`RUDDER_FLY_APP_NAME` (workers app, e.g. `rudder-workers-exla`) rather than
|
|
236
|
+
`FLY_APP_NAME`, which Fly auto-injects with the running app's own name.
|
|
237
|
+
|
|
238
|
+
Deploy with:
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
cd cloud
|
|
242
|
+
flyctl deploy --remote-only
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
`fly.toml` carries the env block; secrets (`BETTER_AUTH_SECRET`,
|
|
246
|
+
`FLY_API_TOKEN`, `GOOGLE_CLIENT_SECRET`, `GITHUB_CLIENT_SECRET`,
|
|
247
|
+
`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) live in `flyctl secrets`.
|
|
248
|
+
|
|
249
|
+
## Fly Machines
|
|
250
|
+
|
|
251
|
+
Rudder Cloud creates one Fly Machine per sail through the Fly Machines API.
|
|
252
|
+
`FLY_APP_NAME` must point at an existing Fly app, and `RUDDER_WORKER_IMAGE`
|
|
253
|
+
should be an image built from `worker/Dockerfile`.
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
docker buildx build --platform linux/amd64,linux/arm64 \
|
|
257
|
+
-f cloud/worker/Dockerfile \
|
|
258
|
+
-t public.ecr.aws/exla/rudder-worker:latest \
|
|
259
|
+
--push .
|
|
260
|
+
|
|
261
|
+
docker buildx build --platform linux/arm64 \
|
|
262
|
+
-f cloud/worker/Dockerfile \
|
|
263
|
+
-t public.ecr.aws/exla/rudder-worker:arm64 \
|
|
264
|
+
--push .
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
When the local docker VM has no room for an amd64 build (it runs under
|
|
268
|
+
emulation on Apple Silicon), build on Fly's remote builder instead and point
|
|
269
|
+
`RUDDER_WORKER_IMAGE` at the Fly registry, which the workers app can pull
|
|
270
|
+
from directly:
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
printf 'app = "rudder-workers-exla"\n' > /tmp/fly-worker.toml
|
|
274
|
+
FLY_API_TOKEN="$(awk '/^access_token:/{print $2}' ~/.fly/config.yml)" \
|
|
275
|
+
flyctl deploy --config /tmp/fly-worker.toml --dockerfile cloud/worker/Dockerfile \
|
|
276
|
+
--build-only --push --remote-only --image-label <version> .
|
|
277
|
+
# -> registry.fly.io/rudder-workers-exla:<version>
|
|
278
|
+
# mirror an arm64 build under the same name so the `:arm64` derivation resolves:
|
|
279
|
+
docker buildx imagetools create -t registry.fly.io/rudder-workers-exla:arm64 \
|
|
280
|
+
public.ecr.aws/exla/rudder-worker:<version>-arm64
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
The image includes Rudder and agent CLIs. Legacy sails restore their snapshot
|
|
284
|
+
and run a finite isolated task; home machines host persistent project dashboards.
|
|
285
|
+
In the initial v4 track, home tasks use automatic planning, while legacy sails
|
|
286
|
+
accept direct execution and refuse discussion/decomposition with a home-workspace
|
|
287
|
+
alternative. Workspace migration and no-task dashboards keep their existing path.
|
|
288
|
+
|
|
289
|
+
Agent CLIs run as the non-root `rudder` user. Bounded Claude/Codex update checks
|
|
290
|
+
run in the background so readiness does not wait on an update server.
|
|
291
|
+
`RUDDER_WORKER_SKIP_AGENT_UPDATE=1` disables them. Home npm/pnpm/uv download
|
|
292
|
+
caches default to the account volume under `/workspace/cache/build`; explicit
|
|
293
|
+
tool paths take precedence. `RUDDER_CACHE_DIR` selects another root and
|
|
294
|
+
`RUDDER_BUILD_CACHE=0` keeps tool defaults. Writable build outputs remain per
|
|
295
|
+
workspace; these download caches are not a distributed action cache.
|
|
296
|
+
|
|
297
|
+
Private previews reuse persisted Tailscale identity, probe live project-owned
|
|
298
|
+
HTTP listeners and remove stale routes. `rudder cloud preview [port]` opens a
|
|
299
|
+
ready project link on a connected laptop; `--list`/`--json` only inspect state.
|
|
300
|
+
Discovery polls every five seconds and first-time certificate setup can take
|
|
301
|
+
longer. The transport uses bounded output batching/backpressure and requests a
|
|
302
|
+
fresh dashboard repaint after a gap rather than replaying an arbitrary byte tail.
|
|
303
|
+
|
|
304
|
+
## Bring Your Own VM
|
|
305
|
+
|
|
306
|
+
Users can run Rudder Cloud workers on their own workstation or server instead
|
|
307
|
+
of Fly Machines:
|
|
308
|
+
|
|
309
|
+
```bash
|
|
310
|
+
rudder cloud login
|
|
311
|
+
rudder cloud setup-byoc rudder-workstation
|
|
312
|
+
rudder cloud "fix the failing migration"
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
`setup-byoc` expects an SSH host that is available from `~/.ssh/config`, uses
|
|
316
|
+
key-based auth, and has Docker available to the SSH user. It stores `byoc` as
|
|
317
|
+
the local default runtime for that CLI login, plus the SSH host for automatic
|
|
318
|
+
worker startup.
|
|
319
|
+
Future `/cloud <task>`, `/sail <task>`, and `rudder cloud <task>` launches
|
|
320
|
+
upload the same snapshot but return a `docker run` command instead of calling
|
|
321
|
+
the Fly Machines API. If an SSH host is configured, the CLI starts that command
|
|
322
|
+
on the host with `nohup`; otherwise it prints the command for manual execution.
|
|
323
|
+
It passes `RUDDER_SNAPSHOT_URL`, `RUDDER_WORKER_TOKEN`, `RUDDER_CLOUD_URL`, and
|
|
324
|
+
task metadata into `cloud/worker/entrypoint.sh`, which already reports
|
|
325
|
+
heartbeats and completion back to the control plane.
|
|
326
|
+
|
|
327
|
+
Useful commands:
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
rudder cloud runtime # show fly or byoc
|
|
331
|
+
rudder cloud runtime fly # switch back to Fly Machines
|
|
332
|
+
rudder cloud byoc "task" # prepare one BYOC run without changing default
|
|
333
|
+
rudder cloud bootstrap <sailId> # regenerate an expired BYOC command
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
Set `RUDDER_BYOC_AUTOSTART=0` to force the CLI to print the Docker command
|
|
337
|
+
instead of starting it over SSH.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Rudder Cloud home host (cloud/host/rudder-host-setup). Agents sandbox
|
|
2
|
+
# themselves with unprivileged user namespaces, and big repos need far more
|
|
3
|
+
# inotify watches and open files than distro defaults. A leading "-" lets
|
|
4
|
+
# systemd-sysctl skip a key this kernel does not have.
|
|
5
|
+
-kernel.apparmor_restrict_unprivileged_userns = 0
|
|
6
|
+
-kernel.unprivileged_userns_clone = 1
|
|
7
|
+
fs.inotify.max_user_watches = 1048576
|
|
8
|
+
fs.inotify.max_user_instances = 8192
|
|
9
|
+
vm.max_map_count = 1048576
|
|
10
|
+
fs.file-max = 4194304
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Applies an image update the home asked for. The supervisor writes
|
|
3
|
+
# desired.json on the volume when the control plane sends it an `update` RPC
|
|
4
|
+
# (cloud/worker/home-host.mjs); rudder-home-update.path runs this as root:
|
|
5
|
+
# pull the image, put the new worker token in home.env, recreate the container.
|
|
6
|
+
#
|
|
7
|
+
# The token is never printed. A request that cannot be applied is set aside as
|
|
8
|
+
# desired.failed.json so the path unit does not retry it in a loop; the control
|
|
9
|
+
# plane keeps accepting the old token until the home connects with the new one.
|
|
10
|
+
set -eu
|
|
11
|
+
umask 077
|
|
12
|
+
|
|
13
|
+
CONF=${RUDDER_HOST_CONF:-/etc/rudder}
|
|
14
|
+
DATA=${RUDDER_HOST_DATA:-/srv/workspace}
|
|
15
|
+
START=${RUDDER_HOME_START:-/usr/local/bin/rudder-home-start}
|
|
16
|
+
DESIRED="$DATA/.rudder-host/desired.json"
|
|
17
|
+
FAILED="$DATA/.rudder-host/desired.failed.json"
|
|
18
|
+
|
|
19
|
+
[ -f "$DESIRED" ] || exit 0
|
|
20
|
+
|
|
21
|
+
fail() {
|
|
22
|
+
echo "rudder-home-apply-update: $1" >&2
|
|
23
|
+
mv -f "$DESIRED" "$FAILED" 2>/dev/null || rm -f "$DESIRED"
|
|
24
|
+
exit 1
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# One field of the single-line object home-host.mjs writes. Both values are
|
|
28
|
+
# checked against strict shapes before docker or the env file sees them.
|
|
29
|
+
field() {
|
|
30
|
+
sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p" "$DESIRED" | head -n 1
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
# An image reference without its tag or digest.
|
|
34
|
+
repository() {
|
|
35
|
+
ref=${1%%@*}
|
|
36
|
+
case ${ref##*/} in
|
|
37
|
+
*:*) ref=${ref%:*} ;;
|
|
38
|
+
esac
|
|
39
|
+
printf '%s\n' "$ref"
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
IMAGE=$(field image)
|
|
43
|
+
TOKEN=$(field workerToken)
|
|
44
|
+
printf '%s\n' "$IMAGE" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9._/:@-]{0,255}' || fail "the requested image is not a valid reference"
|
|
45
|
+
printf '%s\n' "$TOKEN" | grep -Eqx 'rdrw_[A-Za-z0-9_-]{16,200}' || fail "the requested worker token is malformed"
|
|
46
|
+
|
|
47
|
+
# The volume is writable from inside the container, so a request may only move
|
|
48
|
+
# to another tag or digest of the image this host already runs.
|
|
49
|
+
CURRENT=$(cat "$CONF/image" 2>/dev/null || true)
|
|
50
|
+
if [ -n "$CURRENT" ] && [ "$(repository "$IMAGE")" != "$(repository "$CURRENT")" ]; then
|
|
51
|
+
fail "refusing $IMAGE: not the same repository as $CURRENT"
|
|
52
|
+
fi
|
|
53
|
+
|
|
54
|
+
echo "rudder-home-apply-update: pulling $IMAGE"
|
|
55
|
+
docker pull -q "$IMAGE" >/dev/null || fail "docker pull $IMAGE failed (is this host logged in to its registry?)"
|
|
56
|
+
|
|
57
|
+
# Every other line of the env file stays; only the token changes.
|
|
58
|
+
grep -v '^RUDDER_WORKER_TOKEN=' "$CONF/home.env" > "$CONF/home.env.tmp" || true
|
|
59
|
+
printf 'RUDDER_WORKER_TOKEN=%s\n' "$TOKEN" >> "$CONF/home.env.tmp"
|
|
60
|
+
chmod 600 "$CONF/home.env.tmp"
|
|
61
|
+
mv -f "$CONF/home.env.tmp" "$CONF/home.env"
|
|
62
|
+
printf '%s\n' "$IMAGE" > "$CONF/image.tmp"
|
|
63
|
+
mv -f "$CONF/image.tmp" "$CONF/image"
|
|
64
|
+
rm -f "$DESIRED" "$FAILED"
|
|
65
|
+
|
|
66
|
+
"$START"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Runs the Rudder home container on an external host from /etc/rudder/image
|
|
3
|
+
# and /etc/rudder/home.env (0600, written by `rudder cloud host`). Installed by
|
|
4
|
+
# rudder-host-setup; run by rudder-home.service with --keep, and with no flag
|
|
5
|
+
# by rudder-home-apply-update to recreate the container on a new image or token.
|
|
6
|
+
set -eu
|
|
7
|
+
|
|
8
|
+
CONF=${RUDDER_HOST_CONF:-/etc/rudder}
|
|
9
|
+
DATA=${RUDDER_HOST_DATA:-/srv/workspace}
|
|
10
|
+
NAME=rudder-home
|
|
11
|
+
|
|
12
|
+
IMAGE=$(cat "$CONF/image" 2>/dev/null || true)
|
|
13
|
+
[ -n "$IMAGE" ] || { echo "rudder-home-start: $CONF/image is missing; run \`rudder cloud host\` first" >&2; exit 1; }
|
|
14
|
+
[ -s "$CONF/home.env" ] || { echo "rudder-home-start: $CONF/home.env is missing; run \`rudder cloud host\` first" >&2; exit 1; }
|
|
15
|
+
HOST_NAME=$(cat "$CONF/hostname" 2>/dev/null || echo rudder-home)
|
|
16
|
+
|
|
17
|
+
# --keep: a boot after Docker already restarted the container leaves it alone.
|
|
18
|
+
if [ "${1:-}" = "--keep" ] \
|
|
19
|
+
&& [ "$(docker inspect -f '{{.State.Running}} {{.Config.Image}}' "$NAME" 2>/dev/null || true)" = "true $IMAGE" ]; then
|
|
20
|
+
exit 0
|
|
21
|
+
fi
|
|
22
|
+
|
|
23
|
+
if docker container inspect "$NAME" >/dev/null 2>&1; then
|
|
24
|
+
# The supervisor persists every agent as paused on SIGTERM; give it time.
|
|
25
|
+
docker stop -t 90 "$NAME" >/dev/null || true
|
|
26
|
+
docker rm "$NAME" >/dev/null || true
|
|
27
|
+
fi
|
|
28
|
+
|
|
29
|
+
# --privileged: FUSE, swap and the agents' user-namespace sandboxes, as on Fly.
|
|
30
|
+
# /workspace must be its own mount (root-bootstrap.sh checks mountinfo).
|
|
31
|
+
docker run -d --name "$NAME" --restart unless-stopped \
|
|
32
|
+
--privileged --init --hostname "$HOST_NAME" \
|
|
33
|
+
--shm-size 16g --ulimit nofile=1048576:1048576 \
|
|
34
|
+
--log-opt max-size=50m --log-opt max-file=5 \
|
|
35
|
+
--env-file "$CONF/home.env" -e RUDDER_HOST_IMAGE="$IMAGE" \
|
|
36
|
+
-v "$DATA:/workspace" "$IMAGE" >/dev/null
|
|
37
|
+
echo "rudder-home-start: running $IMAGE"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
[Unit]
|
|
2
|
+
Description=Rudder Cloud home (worker container)
|
|
3
|
+
Requires=docker.service
|
|
4
|
+
After=docker.service network-online.target
|
|
5
|
+
Wants=network-online.target
|
|
6
|
+
RequiresMountsFor=/srv/workspace
|
|
7
|
+
|
|
8
|
+
[Service]
|
|
9
|
+
Type=oneshot
|
|
10
|
+
RemainAfterExit=yes
|
|
11
|
+
ExecStart=/usr/local/bin/rudder-home-start --keep
|
|
12
|
+
ExecStop=/usr/bin/docker stop -t 90 rudder-home
|
|
13
|
+
TimeoutStartSec=20min
|
|
14
|
+
TimeoutStopSec=150
|
|
15
|
+
|
|
16
|
+
[Install]
|
|
17
|
+
WantedBy=multi-user.target
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Prepares a Linux host (Ubuntu or Debian, as root) to run a Rudder Cloud home:
|
|
3
|
+
# Docker, rsync, the sysctls agents need, the data disk at /srv/workspace and
|
|
4
|
+
# the systemd units. Safe to run again. It does not start the home: `rudder
|
|
5
|
+
# cloud host` writes /etc/rudder/home.env and /etc/rudder/image, then starts
|
|
6
|
+
# rudder-home.service.
|
|
7
|
+
#
|
|
8
|
+
# sudo ./rudder-host-setup [--data-disk /dev/nvme1n1]
|
|
9
|
+
#
|
|
10
|
+
# The data disk is formatted only when it holds no filesystem and no partitions.
|
|
11
|
+
set -eu
|
|
12
|
+
|
|
13
|
+
HERE=$(cd "$(dirname "$0")" && pwd)
|
|
14
|
+
DATA=/srv/workspace
|
|
15
|
+
LABEL=rudder-ws
|
|
16
|
+
DISK=""
|
|
17
|
+
while [ "$#" -gt 0 ]; do
|
|
18
|
+
case "$1" in
|
|
19
|
+
--data-disk) DISK=${2:?--data-disk needs a device}; shift 2 ;;
|
|
20
|
+
--data-disk=*) DISK=${1#*=}; shift ;;
|
|
21
|
+
-h|--help) sed -n '2,10p' "$0"; exit 0 ;;
|
|
22
|
+
*) echo "rudder-host-setup: unknown argument $1" >&2; exit 64 ;;
|
|
23
|
+
esac
|
|
24
|
+
done
|
|
25
|
+
|
|
26
|
+
[ "$(id -u)" = 0 ] || { echo "rudder-host-setup: run it as root (sudo)" >&2; exit 1; }
|
|
27
|
+
|
|
28
|
+
if ! command -v docker >/dev/null 2>&1 || ! command -v rsync >/dev/null 2>&1; then
|
|
29
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
30
|
+
apt-get update -q
|
|
31
|
+
apt-get install -y -q docker.io rsync
|
|
32
|
+
fi
|
|
33
|
+
systemctl enable --now docker >/dev/null
|
|
34
|
+
|
|
35
|
+
install -m 644 "$HERE/90-rudder.conf" /etc/sysctl.d/90-rudder.conf
|
|
36
|
+
systemctl restart systemd-sysctl.service 2>/dev/null || sysctl -e -q -p /etc/sysctl.d/90-rudder.conf || true
|
|
37
|
+
|
|
38
|
+
mkdir -p "$DATA"
|
|
39
|
+
if [ -n "$DISK" ]; then
|
|
40
|
+
[ -b "$DISK" ] || { echo "rudder-host-setup: $DISK is not a block device" >&2; exit 1; }
|
|
41
|
+
if [ "$(blkid -s LABEL -o value "$DISK" 2>/dev/null || true)" = "$LABEL" ]; then
|
|
42
|
+
: # set up by an earlier run
|
|
43
|
+
elif blkid -p "$DISK" >/dev/null 2>&1 || [ "$(lsblk -n -o NAME "$DISK" | wc -l)" -ne 1 ]; then
|
|
44
|
+
echo "rudder-host-setup: $DISK already holds a filesystem or partitions; not touching it" >&2
|
|
45
|
+
exit 1
|
|
46
|
+
else
|
|
47
|
+
echo "rudder-host-setup: formatting $DISK as ext4 ($LABEL)"
|
|
48
|
+
mkfs.ext4 -q -L "$LABEL" -m 0 "$DISK"
|
|
49
|
+
fi
|
|
50
|
+
grep -q "^LABEL=$LABEL " /etc/fstab || echo "LABEL=$LABEL $DATA ext4 defaults,noatime,nofail 0 2" >> /etc/fstab
|
|
51
|
+
systemctl daemon-reload
|
|
52
|
+
mountpoint -q "$DATA" || mount "$DATA"
|
|
53
|
+
fi
|
|
54
|
+
# The worker runs as uid 1500 (cloud/worker/root-bootstrap.sh).
|
|
55
|
+
chown 1500:1500 "$DATA"
|
|
56
|
+
|
|
57
|
+
mkdir -p /etc/rudder
|
|
58
|
+
chmod 700 /etc/rudder
|
|
59
|
+
install -m 755 "$HERE/rudder-home-start" /usr/local/bin/rudder-home-start
|
|
60
|
+
install -m 755 "$HERE/rudder-home-apply-update" /usr/local/bin/rudder-home-apply-update
|
|
61
|
+
install -m 644 "$HERE/rudder-home.service" "$HERE/rudder-home-update.service" "$HERE/rudder-home-update.path" /etc/systemd/system/
|
|
62
|
+
systemctl daemon-reload
|
|
63
|
+
systemctl enable rudder-home.service rudder-home-update.path >/dev/null 2>&1
|
|
64
|
+
systemctl start rudder-home-update.path
|
|
65
|
+
echo "rudder-host-setup: ready ($DATA: $(df -h --output=size "$DATA" | tail -n 1 | tr -d ' '))"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@exla/rudder-cloud",
|
|
3
|
+
"version": "0.7.7",
|
|
4
|
+
"private": true,
|
|
5
|
+
"license": "Elastic-2.0",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"dev": "tsx src/server.ts",
|
|
9
|
+
"build": "tsc -p tsconfig.json",
|
|
10
|
+
"start": "node dist/server.js"
|
|
11
|
+
},
|
|
12
|
+
"dependencies": {
|
|
13
|
+
"@anthropic-ai/sdk": "1.3.0",
|
|
14
|
+
"@aws-sdk/client-s3": "^3.1047.0",
|
|
15
|
+
"@aws-sdk/credential-provider-web-identity": "^3.972.85",
|
|
16
|
+
"@aws-sdk/s3-request-presigner": "^3.1047.0",
|
|
17
|
+
"@opentelemetry/api": "~1.9.1",
|
|
18
|
+
"@opentelemetry/exporter-metrics-otlp-http": "^0.223.0",
|
|
19
|
+
"@opentelemetry/resources": "^2.12.0",
|
|
20
|
+
"@opentelemetry/sdk-metrics": "^2.12.0",
|
|
21
|
+
"better-auth": "^1.4.0",
|
|
22
|
+
"better-sqlite3": "^12.5.0",
|
|
23
|
+
"pg": "8.23.1",
|
|
24
|
+
"ws": "^8.18.0"
|
|
25
|
+
},
|
|
26
|
+
"devDependencies": {
|
|
27
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
28
|
+
"@types/node": "^22.15.3",
|
|
29
|
+
"@types/pg": "8.23.1",
|
|
30
|
+
"@types/ws": "^8.5.13",
|
|
31
|
+
"tsx": "^4.21.0",
|
|
32
|
+
"typescript": "^5.8.3"
|
|
33
|
+
},
|
|
34
|
+
"engines": {
|
|
35
|
+
"node": ">=20"
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export type HostArgs = {
|
|
2
|
+
target: string;
|
|
3
|
+
dataDisk?: string;
|
|
4
|
+
hostname?: string;
|
|
5
|
+
copyFromFly: boolean;
|
|
6
|
+
skipSetup: boolean;
|
|
7
|
+
yes: boolean;
|
|
8
|
+
};
|
|
9
|
+
export type HostClient = {
|
|
10
|
+
request<T>(pathOrUrl: string, init: {
|
|
11
|
+
method: string;
|
|
12
|
+
body?: unknown;
|
|
13
|
+
}): Promise<T>;
|
|
14
|
+
};
|
|
15
|
+
export declare const HOST_FILES: string[];
|
|
16
|
+
export declare function validImage(value: unknown): value is string;
|
|
17
|
+
export declare function parseHostArgs(args: string[], yes?: boolean): HostArgs;
|
|
18
|
+
/** The env bundle as a Docker --env-file. Refuses anything that would not round-trip. */
|
|
19
|
+
export declare function renderHomeEnv(env: Record<string, unknown>): string;
|
|
20
|
+
/** The manual disk copy from the Fly home, printed by --copy-from-fly. */
|
|
21
|
+
export declare function copyFromFlySteps(target: string, filesDir?: string): string;
|
|
22
|
+
export declare function hostFilesDir(): string;
|
|
23
|
+
export declare function hostCommand(rawArgs: string[], client: HostClient, options: {
|
|
24
|
+
json?: boolean;
|
|
25
|
+
yes?: boolean;
|
|
26
|
+
}): Promise<void>;
|