@agentsky/sky 3.36.0 → 3.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +14 -0
- package/agent-plugins/plugins/browser/skills/agentsky-browser/SKILL.md +1 -0
- package/agent-plugins/plugins/crm/skills/agentsky-crm/SKILL.md +1 -0
- package/agent-plugins/plugins/meetings/skills/agentsky-meetings/SKILL.md +1 -0
- package/agent-plugins/plugins/wechat/skills/agentsky-wechat/SKILL.md +1 -0
- package/dist/{chunk-TLR374RK.js → chunk-QDOMHROS.js} +12 -5
- package/dist/{chunk-5CWYSS5Q.js → chunk-WZEHNEEY.js} +20310 -21806
- package/dist/index.js +2611 -1097
- package/dist/sse-GSK4TBYZ.js +38 -0
- package/dist/{trace-DZZ7QYJT.js → trace-UKKX72AB.js} +2 -4
- package/dist/{trace-library-VZX576JO.js → trace-library-4P37RASU.js} +3 -5
- package/dist/worker-KD5TINHF.js +1150 -0
- package/package.json +5 -5
- package/dist/chunk-WWWRFIK5.js +0 -8
- package/dist/src-HN6NQQYS.js +0 -346
- package/dist/worker-6Q4K4HZW.js +0 -190
|
@@ -0,0 +1,1150 @@
|
|
|
1
|
+
import {
|
|
2
|
+
configPath
|
|
3
|
+
} from "./chunk-XHZJG3QT.js";
|
|
4
|
+
import "./chunk-MLKGABMK.js";
|
|
5
|
+
|
|
6
|
+
// src/worker-docker.ts
|
|
7
|
+
import { closeSync as closeSync2, mkdirSync as mkdirSync2, openSync as openSync2, readFileSync as readFileSync3 } from "fs";
|
|
8
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
9
|
+
import { createHash as createHash3, randomBytes as randomBytes2 } from "crypto";
|
|
10
|
+
|
|
11
|
+
// ../../packages/spec/src/worker-lease.ts
|
|
12
|
+
function parseWorkerLease(value) {
|
|
13
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid worker lease");
|
|
14
|
+
const lease = value;
|
|
15
|
+
for (const field of ["runId", "bindingId", "podId", "executorId"]) {
|
|
16
|
+
if (typeof lease[field] !== "string" || !lease[field].length || lease[field].length > 256) throw new Error("Invalid worker lease identity");
|
|
17
|
+
}
|
|
18
|
+
if (!Number.isSafeInteger(lease.fence) || Number(lease.fence) < 1 || !Number.isSafeInteger(lease.generation) || Number(lease.generation) < 1 || typeof lease.expiresAt !== "string" || !Number.isFinite(Date.parse(lease.expiresAt))) throw new Error("Invalid worker lease deadline");
|
|
19
|
+
return {
|
|
20
|
+
runId: String(lease.runId),
|
|
21
|
+
bindingId: String(lease.bindingId),
|
|
22
|
+
podId: String(lease.podId),
|
|
23
|
+
executorId: String(lease.executorId),
|
|
24
|
+
fence: Number(lease.fence),
|
|
25
|
+
generation: Number(lease.generation),
|
|
26
|
+
expiresAt: lease.expiresAt
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function sameWorkerLeaseIdentity(left, right) {
|
|
30
|
+
return left.runId === right.runId && left.podId === right.podId && left.bindingId === right.bindingId && left.executorId === right.executorId && left.fence === right.fence && left.generation === right.generation;
|
|
31
|
+
}
|
|
32
|
+
function workerLeaseIdentityFollows(current, next) {
|
|
33
|
+
return current.runId === next.runId && current.podId === next.podId && next.fence >= current.fence && next.generation >= current.generation && (next.fence > current.fence || next.executorId === current.executorId) && (next.generation > current.generation || next.bindingId === current.bindingId);
|
|
34
|
+
}
|
|
35
|
+
function canRenewWorkerLease(current, next, now = Date.now()) {
|
|
36
|
+
return workerLeaseIdentityFollows(current, next) && Date.parse(current.expiresAt) > now && Date.parse(next.expiresAt) >= Date.parse(current.expiresAt) && Date.parse(next.expiresAt) > now;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// src/worker-docker-command.ts
|
|
40
|
+
import { spawn } from "child_process";
|
|
41
|
+
var DockerCommandError = class extends Error {
|
|
42
|
+
constructor(args, exitCode, stderr) {
|
|
43
|
+
super(`Docker ${args[0]} failed with exit code ${exitCode}${stderr ? `: ${stderr}` : ""}`);
|
|
44
|
+
this.args = args;
|
|
45
|
+
this.exitCode = exitCode;
|
|
46
|
+
this.stderr = stderr;
|
|
47
|
+
this.name = "DockerCommandError";
|
|
48
|
+
}
|
|
49
|
+
args;
|
|
50
|
+
exitCode;
|
|
51
|
+
stderr;
|
|
52
|
+
};
|
|
53
|
+
function dockerCommand(args, options = {}) {
|
|
54
|
+
if (options.signal?.aborted) return Promise.reject(new Error("Worker Docker command was cancelled"));
|
|
55
|
+
return new Promise((resolve, reject) => {
|
|
56
|
+
const child = spawn("docker", args, { stdio: ["ignore", options.stdoutFile ?? "pipe", "pipe"] });
|
|
57
|
+
let output = "";
|
|
58
|
+
let errors = "";
|
|
59
|
+
child.stdout?.on("data", (chunk) => {
|
|
60
|
+
output = (output + chunk.toString()).slice(-1024 * 1024);
|
|
61
|
+
});
|
|
62
|
+
child.stderr?.on("data", (chunk) => {
|
|
63
|
+
errors = (errors + chunk.toString()).slice(-16 * 1024);
|
|
64
|
+
});
|
|
65
|
+
const abort = () => child.kill("SIGTERM");
|
|
66
|
+
options.signal?.addEventListener("abort", abort, { once: true });
|
|
67
|
+
const timer = options.timeoutMs ? setTimeout(() => child.kill("SIGKILL"), options.timeoutMs) : void 0;
|
|
68
|
+
child.once("error", reject);
|
|
69
|
+
child.once("close", (code) => {
|
|
70
|
+
clearTimeout(timer);
|
|
71
|
+
options.signal?.removeEventListener("abort", abort);
|
|
72
|
+
if (code === 0) resolve(output.trim());
|
|
73
|
+
else reject(new DockerCommandError(args, code, errors.trim()));
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
var NO_SUCH_CONTAINER = /^Error(?: response from daemon)?: No such container: (\S+)$/i;
|
|
78
|
+
var REMOVAL_IN_PROGRESS = /^Error response from daemon: removal of container (\S+) is already in progress$/i;
|
|
79
|
+
var NO_SUCH_OBJECT = /No such (?:object|container)/i;
|
|
80
|
+
async function containerGone(id, timeoutMs) {
|
|
81
|
+
try {
|
|
82
|
+
await dockerCommand(["inspect", "--type", "container", "--format", "{{.Id}}", id], { timeoutMs });
|
|
83
|
+
return false;
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if (error instanceof DockerCommandError && error.exitCode !== null && NO_SUCH_OBJECT.test(error.stderr)) return true;
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
async function removeContainers(ids, options = {}) {
|
|
90
|
+
if (!ids.length) return;
|
|
91
|
+
const deadline = Date.now() + (options.waitMs ?? 3e4);
|
|
92
|
+
const remaining = () => Math.max(1, deadline - Date.now());
|
|
93
|
+
let refusal;
|
|
94
|
+
try {
|
|
95
|
+
await dockerCommand(["rm", "--force", ...ids], { timeoutMs: remaining() });
|
|
96
|
+
return;
|
|
97
|
+
} catch (error) {
|
|
98
|
+
if (!(error instanceof DockerCommandError) || error.exitCode === null) throw error;
|
|
99
|
+
const lines = error.stderr.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
100
|
+
if (!lines.length || !lines.every((line) => NO_SUCH_CONTAINER.test(line) || REMOVAL_IN_PROGRESS.test(line))) throw error;
|
|
101
|
+
refusal = error;
|
|
102
|
+
}
|
|
103
|
+
if (refusal.stderr.split("\n").some((line) => REMOVAL_IN_PROGRESS.test(line.trim()))) {
|
|
104
|
+
let delay = Math.min(options.intervalMs ?? 100, 1e3);
|
|
105
|
+
for (const id of ids) {
|
|
106
|
+
while (true) {
|
|
107
|
+
if (Date.now() >= deadline) throw refusal;
|
|
108
|
+
try {
|
|
109
|
+
if (await containerGone(id, remaining())) break;
|
|
110
|
+
} catch (error) {
|
|
111
|
+
if (error instanceof DockerCommandError && error.exitCode === null && Date.now() >= deadline) throw refusal;
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
if (Date.now() >= deadline) throw refusal;
|
|
115
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(delay, remaining())));
|
|
116
|
+
delay = Math.min(delay * 2, 1e3);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
for (const id of ids) {
|
|
122
|
+
if (!await containerGone(id, remaining())) await dockerCommand(["rm", "--force", id], { timeoutMs: remaining() });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// src/worker-vm.ts
|
|
127
|
+
import { createHash as createHash2 } from "crypto";
|
|
128
|
+
import { existsSync as existsSync2, mkdtempSync, readFileSync as readFileSync2, readdirSync, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
129
|
+
import { tmpdir } from "os";
|
|
130
|
+
import { dirname as dirname2, join as join2 } from "path";
|
|
131
|
+
|
|
132
|
+
// src/worker-vm-program.ts
|
|
133
|
+
function workerVmProgram() {
|
|
134
|
+
return `
|
|
135
|
+
const parseWorkerLease = ${parseWorkerLease.toString()};
|
|
136
|
+
const sameWorkerLeaseIdentity = ${sameWorkerLeaseIdentity.toString()};
|
|
137
|
+
const workerLeaseIdentityFollows = ${workerLeaseIdentityFollows.toString()};
|
|
138
|
+
const canRenewWorkerLease = ${canRenewWorkerLease.toString()};
|
|
139
|
+
` + String.raw`
|
|
140
|
+
const fs = require("node:fs");
|
|
141
|
+
const http = require("node:http");
|
|
142
|
+
const crypto = require("node:crypto");
|
|
143
|
+
const directory = process.env.WORKER_STATE_DIRECTORY || "/state";
|
|
144
|
+
const config = JSON.parse(fs.readFileSync(directory + "/supervisor-config.json", "utf8"));
|
|
145
|
+
if (!/^[a-f0-9]{32}$/.test(config.nonce) || config.name !== "sky-worker-" + config.nonce || !/^[a-f0-9]{64}$/.test(config.token)) throw new Error("Invalid supervisor configuration");
|
|
146
|
+
const statePath = directory + "/supervisor-state.json";
|
|
147
|
+
// Exactly one process may write this journal. A restart of the same container keeps its
|
|
148
|
+
// hostname and takes the lock over; any other process on the same directory is refused.
|
|
149
|
+
const writer = process.env.WORKER_WRITER_ID || require("node:os").hostname();
|
|
150
|
+
function acquireWriter() {
|
|
151
|
+
const path = directory + "/writer.lock";
|
|
152
|
+
try {
|
|
153
|
+
const file = fs.openSync(path, "wx", 0o600);
|
|
154
|
+
try { fs.writeFileSync(file, JSON.stringify({ writer, nonce: config.nonce })); fs.fsyncSync(file); } finally { fs.closeSync(file); }
|
|
155
|
+
} catch (error) {
|
|
156
|
+
if (error.code !== "EEXIST") throw error;
|
|
157
|
+
const owner = JSON.parse(fs.readFileSync(path, "utf8"));
|
|
158
|
+
if (owner.writer !== writer || owner.nonce !== config.nonce) throw new Error("Another lease writer owns this execution");
|
|
159
|
+
}
|
|
160
|
+
const parent = fs.openSync(directory, "r");
|
|
161
|
+
try { fs.fsyncSync(parent); } finally { fs.closeSync(parent); }
|
|
162
|
+
}
|
|
163
|
+
acquireWriter();
|
|
164
|
+
let state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : {
|
|
165
|
+
nonce: config.nonce, name: config.name, phase: "pending", createAdmitted: false, lease: parseWorkerLease(config.lease), observed: [],
|
|
166
|
+
};
|
|
167
|
+
if (state.nonce !== config.nonce || state.name !== config.name) throw new Error("Supervisor journal identity changed");
|
|
168
|
+
let queue = Promise.resolve();
|
|
169
|
+
function serial(work) {
|
|
170
|
+
const pending = queue.then(work);
|
|
171
|
+
queue = pending.catch(() => {});
|
|
172
|
+
return pending;
|
|
173
|
+
}
|
|
174
|
+
// The absolute deadline the target's PID-namespace holder enforces on its own. It only
|
|
175
|
+
// mirrors the committed lease, after the journal, so a crash can shorten but never extend it.
|
|
176
|
+
const deadlineDirectory = directory + "/deadline";
|
|
177
|
+
function publishDeadline(lease) {
|
|
178
|
+
fs.mkdirSync(deadlineDirectory, { recursive: true, mode: 0o755 });
|
|
179
|
+
const temporary = deadlineDirectory + "/at.next";
|
|
180
|
+
const file = fs.openSync(temporary, "w", 0o644);
|
|
181
|
+
try {
|
|
182
|
+
fs.writeFileSync(file, String(Date.parse(lease.expiresAt)));
|
|
183
|
+
if (Number.isInteger(config.uid) && Number.isInteger(config.gid)) fs.fchownSync(file, config.uid, config.gid);
|
|
184
|
+
fs.fsyncSync(file);
|
|
185
|
+
} finally { fs.closeSync(file); }
|
|
186
|
+
fs.renameSync(temporary, deadlineDirectory + "/at");
|
|
187
|
+
const parent = fs.openSync(deadlineDirectory, "r");
|
|
188
|
+
try { fs.fsyncSync(parent); } finally { fs.closeSync(parent); }
|
|
189
|
+
}
|
|
190
|
+
function commit(next) {
|
|
191
|
+
const temporary = statePath + ".next";
|
|
192
|
+
const file = fs.openSync(temporary, "w", 0o600);
|
|
193
|
+
try {
|
|
194
|
+
fs.writeFileSync(file, JSON.stringify(next));
|
|
195
|
+
if (Number.isInteger(config.uid) && Number.isInteger(config.gid)) fs.fchownSync(file, config.uid, config.gid);
|
|
196
|
+
fs.fsyncSync(file);
|
|
197
|
+
} finally { fs.closeSync(file); }
|
|
198
|
+
fs.renameSync(temporary, statePath);
|
|
199
|
+
const parent = fs.openSync(directory, "r");
|
|
200
|
+
try { fs.fsyncSync(parent); } finally { fs.closeSync(parent); }
|
|
201
|
+
state = next;
|
|
202
|
+
// A stopped execution never gets a later deadline than the one it stopped under.
|
|
203
|
+
publishDeadline(next.phase === "stopped" || next.phase === "stopping" ? { expiresAt: new Date(0).toISOString() } : next.lease);
|
|
204
|
+
}
|
|
205
|
+
function engine(method, path) {
|
|
206
|
+
return new Promise((resolve, reject) => {
|
|
207
|
+
const request = http.request({ socketPath: process.env.DOCKER_SOCKET || "/var/run/docker.sock", path, method }, response => {
|
|
208
|
+
const chunks = []; let size = 0;
|
|
209
|
+
response.on("data", chunk => { size += chunk.length; if (size > 2 * 1024 * 1024) request.destroy(new Error("Docker response too large")); else chunks.push(chunk); });
|
|
210
|
+
response.on("end", () => {
|
|
211
|
+
const text = Buffer.concat(chunks).toString();
|
|
212
|
+
if (response.statusCode === 404) return resolve(null);
|
|
213
|
+
if (response.statusCode < 200 || response.statusCode >= 300) return reject(new Error("Docker operation was not confirmed"));
|
|
214
|
+
try { resolve(text ? JSON.parse(text) : null); } catch { reject(new Error("Invalid Docker response")); }
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
request.setTimeout(5000, () => request.destroy(new Error("Docker operation timed out")));
|
|
218
|
+
request.on("error", reject); request.end();
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
async function ownedContainers() {
|
|
222
|
+
const filter = encodeURIComponent(JSON.stringify({ label: ["agentsky.worker.operation=" + config.nonce] }));
|
|
223
|
+
const rows = await engine("GET", "/containers/json?all=1&filters=" + filter);
|
|
224
|
+
if (!Array.isArray(rows)) throw new Error("Docker inventory is unavailable");
|
|
225
|
+
for (const row of rows) {
|
|
226
|
+
const detail = await engine("GET", "/containers/" + row.Id + "/json");
|
|
227
|
+
if (!detail || detail.Id !== row.Id || detail.Name !== "/" + config.name || detail.Config?.Labels?.["agentsky.worker.operation"] !== config.nonce) throw new Error("Container identity differs from its operation");
|
|
228
|
+
}
|
|
229
|
+
return rows.map(row => row.Id);
|
|
230
|
+
}
|
|
231
|
+
async function stop(reason) {
|
|
232
|
+
if (state.phase === "stopped") return;
|
|
233
|
+
const neverCreated = !state.createAdmitted;
|
|
234
|
+
let persistError;
|
|
235
|
+
try { commit({ ...state, phase: "stopping", reason }); } catch (error) { persistError = error; state = { ...state, phase: "stopping", reason }; }
|
|
236
|
+
const ids = await ownedContainers();
|
|
237
|
+
const observed = [...new Set([...state.observed, ...ids])];
|
|
238
|
+
try { commit({ ...state, observed }); } catch (error) { persistError = error; state = { ...state, observed }; }
|
|
239
|
+
for (const id of ids) await engine("DELETE", "/containers/" + id + "?force=1");
|
|
240
|
+
if ((await ownedContainers()).length) throw new Error("Docker still owns a target container");
|
|
241
|
+
await engine("DELETE", "/containers/" + config.name + "-deadline?force=1");
|
|
242
|
+
if (await engine("GET", "/containers/" + config.name + "-deadline/json")) throw new Error("Docker still owns the deadline holder");
|
|
243
|
+
let cid = state.cid;
|
|
244
|
+
if (!cid) {
|
|
245
|
+
try { cid = fs.readFileSync(directory + "/container.id", "utf8").trim(); } catch {}
|
|
246
|
+
}
|
|
247
|
+
if (!neverCreated && (!/^[a-f0-9]{64}$/.test(cid || "") || !observed.includes(cid))) return;
|
|
248
|
+
if (persistError) throw persistError;
|
|
249
|
+
commit({ ...state, phase: "stopped", cid, stoppedAt: new Date().toISOString() });
|
|
250
|
+
}
|
|
251
|
+
async function reconcile() {
|
|
252
|
+
if (state.phase === "stopped") return;
|
|
253
|
+
if (state.phase === "stopping" || Date.parse(state.lease.expiresAt) <= Date.now()) return stop("lease_expired");
|
|
254
|
+
if (state.phase === "running" && state.cid) {
|
|
255
|
+
const detail = await engine("GET", "/containers/" + state.cid + "/json");
|
|
256
|
+
if (!detail || !detail.State?.Running) await stop("container_ended");
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
async function control(action, body) {
|
|
260
|
+
await reconcile();
|
|
261
|
+
if (action === "/status") return state;
|
|
262
|
+
if (action === "/stop") {
|
|
263
|
+
if (!sameWorkerLeaseIdentity(state.lease, parseWorkerLease(body.lease))) throw new Error("Stop identity changed");
|
|
264
|
+
await stop("requested"); return state;
|
|
265
|
+
}
|
|
266
|
+
if (action === "/not-created") {
|
|
267
|
+
if (state.cid || state.observed.length || !["pending", "creating", "stopping"].includes(state.phase)) throw new Error("Creation already crossed its boundary");
|
|
268
|
+
// Only the producer that has not issued Docker create may send this proof.
|
|
269
|
+
commit({ ...state, phase: "stopped", createAdmitted: false, reason: "producer_did_not_create", stoppedAt: new Date().toISOString() });
|
|
270
|
+
return state;
|
|
271
|
+
}
|
|
272
|
+
if (["stopping", "stopped"].includes(state.phase)) throw new Error("Runtime is already stopped");
|
|
273
|
+
if (action === "/renew") {
|
|
274
|
+
const next = parseWorkerLease(body.lease), previous = state.lease;
|
|
275
|
+
if (!canRenewWorkerLease(previous, next)) throw new Error("Lease cannot be renewed");
|
|
276
|
+
// A renewal after the deadline never republishes a later one.
|
|
277
|
+
if (Date.parse(previous.expiresAt) <= Date.now()) { await stop("renewal_missed_deadline"); throw new Error("Renewal missed its deadline"); }
|
|
278
|
+
commit({ ...state, lease: next });
|
|
279
|
+
return state;
|
|
280
|
+
}
|
|
281
|
+
if (action === "/begin") {
|
|
282
|
+
if (state.phase !== "pending") throw new Error("Creation was already admitted");
|
|
283
|
+
commit({ ...state, phase: "creating", createAdmitted: true });
|
|
284
|
+
} else if (action === "/created") {
|
|
285
|
+
if (!/^[a-f0-9]{64}$/.test(body.cid || "")) throw new Error("Missing container identity");
|
|
286
|
+
const ids = await ownedContainers();
|
|
287
|
+
if (!ids.includes(body.cid)) throw new Error("Created container is outside its operation");
|
|
288
|
+
commit({ ...state, cid: body.cid, observed: [...new Set([...state.observed, ...ids])], phase: "prepared" });
|
|
289
|
+
} else if (action === "/started") {
|
|
290
|
+
if (!state.cid || !(await ownedContainers()).includes(state.cid)) throw new Error("Started container identity changed");
|
|
291
|
+
const detail = await engine("GET", "/containers/" + state.cid + "/json");
|
|
292
|
+
if (!detail?.State?.Running) throw new Error("Target container is not running");
|
|
293
|
+
commit({ ...state, phase: "running" });
|
|
294
|
+
} else throw new Error("Unknown supervisor operation");
|
|
295
|
+
if (Date.parse(state.lease.expiresAt) <= Date.now()) { await stop("lease_expired"); throw new Error("Lease expired before acknowledgement"); }
|
|
296
|
+
return state;
|
|
297
|
+
}
|
|
298
|
+
function authenticated(request) {
|
|
299
|
+
const actual = Buffer.from(request.headers.authorization || "");
|
|
300
|
+
const expected = Buffer.from("Bearer " + config.token);
|
|
301
|
+
return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
|
|
302
|
+
}
|
|
303
|
+
async function serve() {
|
|
304
|
+
commit(state);
|
|
305
|
+
await serial(reconcile);
|
|
306
|
+
const server = http.createServer(async (request, response) => {
|
|
307
|
+
if (!authenticated(request)) { response.writeHead(403).end(); return; }
|
|
308
|
+
try {
|
|
309
|
+
const chunks = []; let size = 0;
|
|
310
|
+
for await (const chunk of request) { size += chunk.length; if (size > 16384) throw new Error("Supervisor request too large"); chunks.push(chunk); }
|
|
311
|
+
const body = size ? JSON.parse(Buffer.concat(chunks).toString()) : {};
|
|
312
|
+
const result = await serial(() => control(request.url, body));
|
|
313
|
+
response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(result));
|
|
314
|
+
} catch {
|
|
315
|
+
response.writeHead(409, { "content-type": "application/json" }).end(JSON.stringify({ error: "Worker supervisor could not confirm the requested operation" }));
|
|
316
|
+
}
|
|
317
|
+
});
|
|
318
|
+
server.listen(Number(process.env.WORKER_CONTROL_PORT || 8787), process.env.WORKER_CONTROL_HOST || "0.0.0.0");
|
|
319
|
+
let queued = false;
|
|
320
|
+
setInterval(() => {
|
|
321
|
+
if (queued) return;
|
|
322
|
+
queued = true;
|
|
323
|
+
void serial(reconcile).catch(() => {}).finally(() => { queued = false; });
|
|
324
|
+
}, 200);
|
|
325
|
+
}
|
|
326
|
+
void serve().catch(() => process.exit(1));
|
|
327
|
+
`;
|
|
328
|
+
}
|
|
329
|
+
function workerDeadlineProgram() {
|
|
330
|
+
return String.raw`
|
|
331
|
+
const fs = require("node:fs");
|
|
332
|
+
const read = () => { try { return Number(fs.readFileSync("/deadline/at", "utf8")); } catch { return 0; } };
|
|
333
|
+
const tick = () => {
|
|
334
|
+
const at = read();
|
|
335
|
+
if (!Number.isFinite(at) || at <= Date.now()) process.exit(0);
|
|
336
|
+
setTimeout(tick, Math.min(1000, at - Date.now()));
|
|
337
|
+
};
|
|
338
|
+
tick();
|
|
339
|
+
`;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// src/worker-watchdog.ts
|
|
343
|
+
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "fs";
|
|
344
|
+
import { dirname, join } from "path";
|
|
345
|
+
import { createHash, randomBytes } from "crypto";
|
|
346
|
+
function writeWorkerJson(path, value) {
|
|
347
|
+
const temporary = `${path}.${randomBytes(6).toString("hex")}.next`;
|
|
348
|
+
const file = openSync(temporary, "w", 384);
|
|
349
|
+
try {
|
|
350
|
+
writeFileSync(file, JSON.stringify(value));
|
|
351
|
+
fsyncSync(file);
|
|
352
|
+
} finally {
|
|
353
|
+
closeSync(file);
|
|
354
|
+
}
|
|
355
|
+
renameSync(temporary, path);
|
|
356
|
+
const directory = openSync(dirname(path), "r");
|
|
357
|
+
try {
|
|
358
|
+
fsyncSync(directory);
|
|
359
|
+
} finally {
|
|
360
|
+
closeSync(directory);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
function recordRetiredWorkerLease(root, lease) {
|
|
364
|
+
const directory = join(root, "retired");
|
|
365
|
+
mkdirSync(directory, { recursive: true, mode: 448 });
|
|
366
|
+
writeWorkerJson(join(directory, createHash("sha256").update(lease.podId).digest("hex") + ".json"), lease);
|
|
367
|
+
}
|
|
368
|
+
function retiredWorkerLease(root, podId) {
|
|
369
|
+
const path = join(root, "retired", createHash("sha256").update(podId).digest("hex") + ".json");
|
|
370
|
+
return existsSync(path) ? parseWorkerLease(JSON.parse(readFileSync(path, "utf8"))) : null;
|
|
371
|
+
}
|
|
372
|
+
function readWorkerIdentity(root, poolId) {
|
|
373
|
+
const path = join(root, "identity.json");
|
|
374
|
+
if (!existsSync(path)) return null;
|
|
375
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
376
|
+
if (typeof value.poolId !== "string" || typeof value.workerId !== "string" || typeof value.resumeToken !== "string" || !Number.isSafeInteger(value.epoch) || Number(value.epoch) < 1) throw new Error("Worker identity file is invalid");
|
|
377
|
+
return value.poolId === poolId ? value : null;
|
|
378
|
+
}
|
|
379
|
+
function writeWorkerIdentity(root, identity) {
|
|
380
|
+
mkdirSync(root, { recursive: true, mode: 448 });
|
|
381
|
+
writeWorkerJson(join(root, "identity.json"), identity);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// src/worker-vm.ts
|
|
385
|
+
var BASE_IMAGE = "node:22.22.0-alpine";
|
|
386
|
+
function workerRootLabel(root) {
|
|
387
|
+
return `agentsky.worker.root=${createHash2("sha256").update(root).digest("hex").slice(0, 32)}`;
|
|
388
|
+
}
|
|
389
|
+
async function labelledContainers(label) {
|
|
390
|
+
const output = await dockerCommand(["ps", "--all", "--quiet", "--no-trunc", "--filter", `label=${label}`]);
|
|
391
|
+
return output.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
392
|
+
}
|
|
393
|
+
async function removeOperationContainers(nonce) {
|
|
394
|
+
for (const label of [`agentsky.worker.operation=${nonce}`, `agentsky.worker.deadline=${nonce}`, `agentsky.worker.supervisor=${nonce}`]) {
|
|
395
|
+
const ids = await labelledContainers(label);
|
|
396
|
+
await removeContainers(ids);
|
|
397
|
+
if ((await labelledContainers(label)).length) throw new Error("Docker still owns a worker container");
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
async function ensureWorkerVmImage() {
|
|
401
|
+
const program = workerVmProgram();
|
|
402
|
+
const hash = createHash2("sha256").update(BASE_IMAGE + program).digest("hex");
|
|
403
|
+
const image = `agentsky-worker-watchdog:${hash}`;
|
|
404
|
+
try {
|
|
405
|
+
await dockerCommand(["image", "inspect", image]);
|
|
406
|
+
return image;
|
|
407
|
+
} catch {
|
|
408
|
+
}
|
|
409
|
+
const directory = mkdtempSync(join2(tmpdir(), "sky-watchdog-build-"));
|
|
410
|
+
try {
|
|
411
|
+
writeFileSync2(join2(directory, "watchdog.cjs"), program);
|
|
412
|
+
writeFileSync2(join2(directory, "Dockerfile"), `FROM ${BASE_IMAGE}
|
|
413
|
+
COPY watchdog.cjs /watchdog.cjs
|
|
414
|
+
ENTRYPOINT ["node", "/watchdog.cjs"]
|
|
415
|
+
`);
|
|
416
|
+
await dockerCommand(["build", "--tag", image, directory]);
|
|
417
|
+
return image;
|
|
418
|
+
} finally {
|
|
419
|
+
rmSync(directory, { recursive: true, force: true });
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
var WorkerVm = class {
|
|
423
|
+
constructor(directory, config, port) {
|
|
424
|
+
this.directory = directory;
|
|
425
|
+
this.config = config;
|
|
426
|
+
this.port = port;
|
|
427
|
+
}
|
|
428
|
+
directory;
|
|
429
|
+
config;
|
|
430
|
+
port;
|
|
431
|
+
async request(action, body = {}) {
|
|
432
|
+
const until = Date.now() + 5e3;
|
|
433
|
+
let response;
|
|
434
|
+
for (; ; ) {
|
|
435
|
+
try {
|
|
436
|
+
response = await fetch(`http://127.0.0.1:${this.port}${action}`, {
|
|
437
|
+
method: "POST",
|
|
438
|
+
headers: { authorization: `Bearer ${this.config.token}`, "content-type": "application/json" },
|
|
439
|
+
body: JSON.stringify(body),
|
|
440
|
+
signal: AbortSignal.timeout(1e3)
|
|
441
|
+
});
|
|
442
|
+
break;
|
|
443
|
+
} catch (error) {
|
|
444
|
+
if (Date.now() >= until) throw error;
|
|
445
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
if (!response.ok) throw new Error("VM watchdog did not confirm the requested operation");
|
|
449
|
+
const state = await response.json();
|
|
450
|
+
if (state.nonce !== this.config.nonce || state.name !== this.config.name) throw new Error("VM watchdog identity changed");
|
|
451
|
+
parseWorkerLease(state.lease);
|
|
452
|
+
return state;
|
|
453
|
+
}
|
|
454
|
+
async stop() {
|
|
455
|
+
let state = await this.request("/status");
|
|
456
|
+
const until = Date.now() + 15e3;
|
|
457
|
+
while (state.phase !== "stopped") {
|
|
458
|
+
state = await this.request("/stop", { lease: state.lease });
|
|
459
|
+
if (state.phase === "stopped") break;
|
|
460
|
+
if (Date.now() >= until) throw new Error("VM watchdog retains an unconfirmed Docker operation");
|
|
461
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
462
|
+
}
|
|
463
|
+
return state;
|
|
464
|
+
}
|
|
465
|
+
async remove() {
|
|
466
|
+
const state = await this.request("/status");
|
|
467
|
+
if (state.phase !== "stopped" || !state.stoppedAt) throw new Error("Cannot remove an active VM watchdog");
|
|
468
|
+
recordRetiredWorkerLease(dirname2(this.directory), state.lease);
|
|
469
|
+
await removeContainers([`${this.config.name}-watchdog`]);
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
async function connectWorkerVm(directory, image) {
|
|
473
|
+
const config = JSON.parse(readFileSync2(join2(directory, "supervisor-config.json"), "utf8"));
|
|
474
|
+
if (!/^[a-f0-9]{32}$/.test(config.nonce) || config.name !== `sky-worker-${config.nonce}` || !/^[a-f0-9]{64}$/.test(config.token)) throw new Error("Invalid worker VM identity");
|
|
475
|
+
const name = `${config.name}-watchdog`;
|
|
476
|
+
try {
|
|
477
|
+
await dockerCommand(["inspect", name]);
|
|
478
|
+
} catch {
|
|
479
|
+
rmSync(join2(directory, "writer.lock"), { force: true });
|
|
480
|
+
await dockerCommand([
|
|
481
|
+
"create",
|
|
482
|
+
"--name",
|
|
483
|
+
name,
|
|
484
|
+
"--restart=always",
|
|
485
|
+
"--label",
|
|
486
|
+
`agentsky.worker.supervisor=${config.nonce}`,
|
|
487
|
+
"--label",
|
|
488
|
+
workerRootLabel(dirname2(directory)),
|
|
489
|
+
"--mount",
|
|
490
|
+
"type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock",
|
|
491
|
+
"--mount",
|
|
492
|
+
`type=bind,source=${directory},target=/state`,
|
|
493
|
+
"--publish",
|
|
494
|
+
"127.0.0.1::8787",
|
|
495
|
+
image ?? await ensureWorkerVmImage()
|
|
496
|
+
]);
|
|
497
|
+
}
|
|
498
|
+
await dockerCommand(["start", name]);
|
|
499
|
+
const raw = await dockerCommand(["inspect", "--format", '{{(index (index .NetworkSettings.Ports "8787/tcp") 0).HostPort}}', name]);
|
|
500
|
+
const port = Number(raw);
|
|
501
|
+
if (!Number.isSafeInteger(port) || port < 1 || port > 65535) throw new Error("VM watchdog has no loopback control port");
|
|
502
|
+
const vm = new WorkerVm(directory, config, port);
|
|
503
|
+
const until = Date.now() + 1e4;
|
|
504
|
+
for (; ; ) {
|
|
505
|
+
try {
|
|
506
|
+
await vm.request("/status");
|
|
507
|
+
return vm;
|
|
508
|
+
} catch (error) {
|
|
509
|
+
if (Date.now() >= until) throw error;
|
|
510
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
async function recoverWorkerVms(root) {
|
|
515
|
+
const image = await ensureWorkerVmImage();
|
|
516
|
+
let adopted = null;
|
|
517
|
+
const known = /* @__PURE__ */ new Set();
|
|
518
|
+
if (existsSync2(root)) for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
519
|
+
if (!entry.isDirectory()) continue;
|
|
520
|
+
const directory = join2(root, entry.name);
|
|
521
|
+
if (!existsSync2(join2(directory, "supervisor-config.json"))) continue;
|
|
522
|
+
const vm = await connectWorkerVm(directory, image);
|
|
523
|
+
const state = await vm.request("/status");
|
|
524
|
+
if (state.phase !== "stopped" && Date.parse(state.lease.expiresAt) > Date.now() && !adopted) {
|
|
525
|
+
adopted = { directory, lease: parseWorkerLease(state.lease), vm };
|
|
526
|
+
known.add(vm.config.nonce);
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
await vm.stop();
|
|
530
|
+
await vm.remove();
|
|
531
|
+
await removeOperationContainers(vm.config.nonce);
|
|
532
|
+
rmSync(directory, { recursive: true, force: true });
|
|
533
|
+
}
|
|
534
|
+
for (const id of await labelledContainers(workerRootLabel(root))) {
|
|
535
|
+
const [detail] = JSON.parse(await dockerCommand(["inspect", id]));
|
|
536
|
+
const labels = detail?.Config?.Labels ?? {};
|
|
537
|
+
const nonce = labels["agentsky.worker.operation"] ?? labels["agentsky.worker.deadline"] ?? labels["agentsky.worker.supervisor"];
|
|
538
|
+
if (nonce && known.has(nonce)) continue;
|
|
539
|
+
await removeContainers([id]);
|
|
540
|
+
}
|
|
541
|
+
const remaining = await labelledContainers(workerRootLabel(root));
|
|
542
|
+
for (const id of remaining) {
|
|
543
|
+
const [detail] = JSON.parse(await dockerCommand(["inspect", id]));
|
|
544
|
+
const labels = detail?.Config?.Labels ?? {};
|
|
545
|
+
if (!known.has(labels["agentsky.worker.operation"] ?? labels["agentsky.worker.deadline"] ?? labels["agentsky.worker.supervisor"] ?? "")) throw new Error("An unknown worker container still occupies this machine");
|
|
546
|
+
}
|
|
547
|
+
return adopted;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// src/worker-docker.ts
|
|
551
|
+
var WorkerProcessError = class extends Error {
|
|
552
|
+
cleanupConfirmed = true;
|
|
553
|
+
};
|
|
554
|
+
var RealDockerRunner = class {
|
|
555
|
+
independentLease = true;
|
|
556
|
+
supervisors = /* @__PURE__ */ new Map();
|
|
557
|
+
async recover(root) {
|
|
558
|
+
const recovered = await recoverWorkerVms(root);
|
|
559
|
+
if (!recovered) return null;
|
|
560
|
+
this.supervisors.set(join3(recovered.directory, "lease.json"), recovered.vm);
|
|
561
|
+
return { directory: recovered.directory, lease: recovered.lease };
|
|
562
|
+
}
|
|
563
|
+
async renewLease(deadlineFile, lease) {
|
|
564
|
+
const supervisor = this.supervisors.get(deadlineFile);
|
|
565
|
+
if (!supervisor) throw new Error("Worker supervisor is unavailable");
|
|
566
|
+
const state = await supervisor.request("/renew", { lease });
|
|
567
|
+
if (state.phase === "stopping" || state.phase === "stopped" || state.lease.expiresAt !== lease.expiresAt || !sameWorkerLeaseIdentity(state.lease, lease)) throw new Error("Worker lease was not accepted");
|
|
568
|
+
}
|
|
569
|
+
/** Resumes supervision of an execution that a previous worker process accepted. */
|
|
570
|
+
async adopt(directory, signal) {
|
|
571
|
+
const leasePath = join3(directory, "lease.json");
|
|
572
|
+
const supervisor = this.supervisors.get(leasePath) ?? await connectWorkerVm(directory);
|
|
573
|
+
this.supervisors.set(leasePath, supervisor);
|
|
574
|
+
await this.supervise(supervisor, leasePath, signal, true, async () => {
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
async run({ image, envFile, port, signal, deadlineFile, onReady }) {
|
|
578
|
+
const directory = dirname3(envFile), nonce = randomBytes2(16).toString("hex"), name = `sky-worker-${nonce}`;
|
|
579
|
+
const leasePath = deadlineFile ?? join3(directory, "lease.json");
|
|
580
|
+
const lease = deadlineFile ? parseWorkerLease(JSON.parse(readFileSync3(deadlineFile, "utf8"))) : {
|
|
581
|
+
runId: name,
|
|
582
|
+
bindingId: name,
|
|
583
|
+
podId: name,
|
|
584
|
+
executorId: name,
|
|
585
|
+
fence: 1,
|
|
586
|
+
generation: 1,
|
|
587
|
+
expiresAt: new Date(Date.now() + 3e4).toISOString()
|
|
588
|
+
};
|
|
589
|
+
writeWorkerJson(join3(directory, "supervisor-config.json"), {
|
|
590
|
+
nonce,
|
|
591
|
+
name,
|
|
592
|
+
token: randomBytes2(32).toString("hex"),
|
|
593
|
+
lease,
|
|
594
|
+
uid: process.getuid?.() ?? 0,
|
|
595
|
+
gid: process.getgid?.() ?? 0
|
|
596
|
+
});
|
|
597
|
+
mkdirSync2(join3(directory, "deadline"), { recursive: true, mode: 493 });
|
|
598
|
+
const supervisor = await connectWorkerVm(directory, await ensureWorkerVmImage());
|
|
599
|
+
this.supervisors.set(leasePath, supervisor);
|
|
600
|
+
await this.supervise(supervisor, leasePath, signal, !!deadlineFile, async () => {
|
|
601
|
+
onReady?.();
|
|
602
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(image)) await dockerCommand(["pull", image], { signal });
|
|
603
|
+
if (signal.aborted || (await supervisor.request("/status")).phase === "stopped") return false;
|
|
604
|
+
await supervisor.request("/begin");
|
|
605
|
+
await dockerCommand([
|
|
606
|
+
"create",
|
|
607
|
+
"--name",
|
|
608
|
+
`${name}-deadline`,
|
|
609
|
+
"--restart=no",
|
|
610
|
+
"--label",
|
|
611
|
+
`agentsky.worker.deadline=${nonce}`,
|
|
612
|
+
"--label",
|
|
613
|
+
workerRootLabel(dirname3(directory)),
|
|
614
|
+
"--network",
|
|
615
|
+
"none",
|
|
616
|
+
"--read-only",
|
|
617
|
+
"--cap-drop",
|
|
618
|
+
"ALL",
|
|
619
|
+
"--security-opt",
|
|
620
|
+
"no-new-privileges",
|
|
621
|
+
"--mount",
|
|
622
|
+
`type=bind,source=${join3(directory, "deadline")},target=/deadline,readonly`,
|
|
623
|
+
"--entrypoint",
|
|
624
|
+
"node",
|
|
625
|
+
await ensureWorkerVmImage(),
|
|
626
|
+
"-e",
|
|
627
|
+
workerDeadlineProgram()
|
|
628
|
+
], { timeoutMs: 15e3 });
|
|
629
|
+
await dockerCommand(["start", `${name}-deadline`]);
|
|
630
|
+
const cidFile = openSync2(join3(directory, "container.id"), "w", 384);
|
|
631
|
+
try {
|
|
632
|
+
const volume = deadlineFile ? `sky-worker-${createHash3("sha256").update(lease.podId).digest("hex")}` : void 0;
|
|
633
|
+
await dockerCommand(
|
|
634
|
+
[
|
|
635
|
+
"create",
|
|
636
|
+
"--name",
|
|
637
|
+
name,
|
|
638
|
+
"--restart=no",
|
|
639
|
+
"--pid",
|
|
640
|
+
`container:${name}-deadline`,
|
|
641
|
+
"--label",
|
|
642
|
+
`agentsky.worker.operation=${nonce}`,
|
|
643
|
+
"--label",
|
|
644
|
+
workerRootLabel(dirname3(directory)),
|
|
645
|
+
`--env-file=${envFile}`,
|
|
646
|
+
"--publish",
|
|
647
|
+
`127.0.0.1:${port}:8080`,
|
|
648
|
+
...volume ? ["--mount", `type=volume,source=${volume},target=/data`] : [],
|
|
649
|
+
image
|
|
650
|
+
],
|
|
651
|
+
{ stdoutFile: cidFile, timeoutMs: 15e3 }
|
|
652
|
+
);
|
|
653
|
+
} finally {
|
|
654
|
+
closeSync2(cidFile);
|
|
655
|
+
}
|
|
656
|
+
const cid = readFileSync3(join3(directory, "container.id"), "utf8").trim();
|
|
657
|
+
await supervisor.request("/created", { cid });
|
|
658
|
+
if (signal.aborted) return true;
|
|
659
|
+
await dockerCommand(["start", name]);
|
|
660
|
+
await supervisor.request("/started");
|
|
661
|
+
return true;
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* One loop for new and adopted executions. While the VM supervisor answers, it alone
|
|
666
|
+
* decides expiry. If it is lost, this host process removes the target itself and only
|
|
667
|
+
* reports a confirmed cleanup when Docker no longer lists any container of the operation.
|
|
668
|
+
*/
|
|
669
|
+
async supervise(supervisor, leasePath, signal, leased, launch) {
|
|
670
|
+
const abort = () => {
|
|
671
|
+
void supervisor.stop().catch(() => {
|
|
672
|
+
});
|
|
673
|
+
};
|
|
674
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
675
|
+
let failure;
|
|
676
|
+
let createIssued = false;
|
|
677
|
+
try {
|
|
678
|
+
const outcome = await launch().then((value) => {
|
|
679
|
+
createIssued = value !== false;
|
|
680
|
+
return value;
|
|
681
|
+
}, (error) => {
|
|
682
|
+
createIssued = true;
|
|
683
|
+
throw error;
|
|
684
|
+
});
|
|
685
|
+
if (outcome === false) createIssued = false;
|
|
686
|
+
if (!signal.aborted) {
|
|
687
|
+
let renewedAt = Date.now();
|
|
688
|
+
for (; ; ) {
|
|
689
|
+
const state = await supervisor.request("/status");
|
|
690
|
+
if (state.phase === "stopped") break;
|
|
691
|
+
if (signal.aborted) {
|
|
692
|
+
await supervisor.stop();
|
|
693
|
+
break;
|
|
694
|
+
}
|
|
695
|
+
if (!leased && Date.now() - renewedAt >= 5e3) {
|
|
696
|
+
await supervisor.request("/renew", { lease: { ...state.lease, expiresAt: new Date(Date.now() + 3e4).toISOString() } });
|
|
697
|
+
renewedAt = Date.now();
|
|
698
|
+
}
|
|
699
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
} catch (error) {
|
|
703
|
+
failure = error;
|
|
704
|
+
} finally {
|
|
705
|
+
signal.removeEventListener("abort", abort);
|
|
706
|
+
try {
|
|
707
|
+
if (!createIssued) await supervisor.request("/not-created").catch(() => {
|
|
708
|
+
});
|
|
709
|
+
await supervisor.stop();
|
|
710
|
+
await supervisor.remove();
|
|
711
|
+
} catch (error) {
|
|
712
|
+
await removeOperationContainers(supervisor.config.nonce);
|
|
713
|
+
failure ??= error;
|
|
714
|
+
}
|
|
715
|
+
this.supervisors.delete(leasePath);
|
|
716
|
+
}
|
|
717
|
+
if (failure && !signal.aborted) throw new WorkerProcessError("Worker container process failed", { cause: failure });
|
|
718
|
+
}
|
|
719
|
+
};
|
|
720
|
+
|
|
721
|
+
// src/worker-runtime.ts
|
|
722
|
+
import { createHash as createHash4, randomBytes as randomBytes3 } from "crypto";
|
|
723
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, rmSync as rmSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
724
|
+
import { dirname as dirname4, join as join4 } from "path";
|
|
725
|
+
function processAlive(pid) {
|
|
726
|
+
try {
|
|
727
|
+
process.kill(pid, 0);
|
|
728
|
+
return true;
|
|
729
|
+
} catch (error) {
|
|
730
|
+
return error.code === "EPERM";
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
var WorkerRuntime = class {
|
|
734
|
+
constructor(docker, port, stateDirectory = join4(dirname4(configPath()), "worker-runtimes")) {
|
|
735
|
+
this.docker = docker;
|
|
736
|
+
this.port = port;
|
|
737
|
+
this.stateDirectory = stateDirectory;
|
|
738
|
+
}
|
|
739
|
+
docker;
|
|
740
|
+
port;
|
|
741
|
+
stateDirectory;
|
|
742
|
+
current = null;
|
|
743
|
+
locked = false;
|
|
744
|
+
/**
|
|
745
|
+
* Takes the state directory for this process and resumes any retained execution.
|
|
746
|
+
* Capacity is only advertised after this returns.
|
|
747
|
+
*/
|
|
748
|
+
async recover() {
|
|
749
|
+
this.lock();
|
|
750
|
+
const recovered = await this.docker.recover?.(this.stateDirectory);
|
|
751
|
+
if (!recovered) return;
|
|
752
|
+
if (!this.docker.adopt) throw new Error("Worker runner cannot resume a retained execution");
|
|
753
|
+
const metadata = JSON.parse(readFileSync4(join4(recovered.directory, "runtime.json"), "utf8"));
|
|
754
|
+
const lease = parseWorkerLease(recovered.lease);
|
|
755
|
+
const running = {
|
|
756
|
+
spec: { sessionId: metadata.sessionId, image: metadata.image, env: {}, lease },
|
|
757
|
+
fingerprint: metadata.fingerprint,
|
|
758
|
+
controller: new AbortController(),
|
|
759
|
+
completion: Promise.resolve(),
|
|
760
|
+
directory: recovered.directory,
|
|
761
|
+
stopping: false
|
|
762
|
+
};
|
|
763
|
+
this.current = running;
|
|
764
|
+
running.completion = this.docker.adopt(recovered.directory, running.controller.signal).then(() => this.finished(running), (error) => {
|
|
765
|
+
if (error instanceof WorkerProcessError) this.finished(running);
|
|
766
|
+
throw error;
|
|
767
|
+
});
|
|
768
|
+
void running.completion.catch(() => {
|
|
769
|
+
});
|
|
770
|
+
this.scheduleDeadline(running);
|
|
771
|
+
}
|
|
772
|
+
/** Releases the state directory; a later process may recover what remains in it. */
|
|
773
|
+
release() {
|
|
774
|
+
if (!this.locked) return;
|
|
775
|
+
const path = join4(this.stateDirectory, "worker.lock");
|
|
776
|
+
try {
|
|
777
|
+
if (JSON.parse(readFileSync4(path, "utf8")).pid === process.pid) rmSync2(path, { force: true });
|
|
778
|
+
} catch {
|
|
779
|
+
}
|
|
780
|
+
this.locked = false;
|
|
781
|
+
}
|
|
782
|
+
lock() {
|
|
783
|
+
if (this.locked) return;
|
|
784
|
+
mkdirSync3(this.stateDirectory, { recursive: true, mode: 448 });
|
|
785
|
+
const path = join4(this.stateDirectory, "worker.lock");
|
|
786
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
787
|
+
try {
|
|
788
|
+
writeFileSync3(path, JSON.stringify({ pid: process.pid }), { flag: "wx", mode: 384 });
|
|
789
|
+
this.locked = true;
|
|
790
|
+
return;
|
|
791
|
+
} catch (error) {
|
|
792
|
+
if (error.code !== "EEXIST") throw error;
|
|
793
|
+
let owner = 0;
|
|
794
|
+
try {
|
|
795
|
+
owner = Number(JSON.parse(readFileSync4(path, "utf8")).pid);
|
|
796
|
+
} catch {
|
|
797
|
+
}
|
|
798
|
+
if (owner > 0 && owner !== process.pid && processAlive(owner)) throw new Error("Another sky worker owns this state directory");
|
|
799
|
+
rmSync2(path, { force: true });
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
throw new Error("Worker state directory lock is contended");
|
|
803
|
+
}
|
|
804
|
+
/** The private state directory this runtime owns while it holds the worker lock. */
|
|
805
|
+
get root() {
|
|
806
|
+
return this.stateDirectory;
|
|
807
|
+
}
|
|
808
|
+
get lease() {
|
|
809
|
+
return this.current?.spec.lease;
|
|
810
|
+
}
|
|
811
|
+
get active() {
|
|
812
|
+
return this.current !== null;
|
|
813
|
+
}
|
|
814
|
+
async start(spec) {
|
|
815
|
+
if (spec.lease) {
|
|
816
|
+
spec = { ...spec, lease: parseWorkerLease(spec.lease) };
|
|
817
|
+
const retired = retiredWorkerLease(this.stateDirectory, spec.lease.podId);
|
|
818
|
+
const newer = retired && workerLeaseIdentityFollows(retired, spec.lease) && (spec.lease.fence > retired.fence || spec.lease.generation > retired.generation);
|
|
819
|
+
if (Date.parse(spec.lease.expiresAt) <= Date.now() || retired && !newer) throw new Error("Worker start lease is expired or retired");
|
|
820
|
+
}
|
|
821
|
+
const fingerprint = createHash4("sha256").update(JSON.stringify([spec.sessionId, spec.image, Object.entries(spec.env).sort()])).digest("hex");
|
|
822
|
+
if (this.current) {
|
|
823
|
+
if (spec.lease && this.current.spec.lease && sameWorkerLeaseIdentity(spec.lease, this.current.spec.lease) && spec.lease.expiresAt === this.current.spec.lease.expiresAt && fingerprint === this.current.fingerprint && !this.current.stopping) return;
|
|
824
|
+
if (spec.lease || this.current.spec.lease) throw new Error("Worker already owns another execution");
|
|
825
|
+
await this.stop();
|
|
826
|
+
}
|
|
827
|
+
for (const [key, value] of Object.entries(spec.env)) {
|
|
828
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || typeof value !== "string" || value.includes("\0")) throw new Error("Invalid worker environment");
|
|
829
|
+
}
|
|
830
|
+
const directory = join4(this.stateDirectory, `sky-worker-${randomBytes3(12).toString("hex")}`);
|
|
831
|
+
const envFile = join4(directory, "pod.env");
|
|
832
|
+
const running = { spec, fingerprint, controller: new AbortController(), completion: Promise.resolve(), directory, stopping: false };
|
|
833
|
+
try {
|
|
834
|
+
mkdirSync3(directory, { mode: 448, recursive: true });
|
|
835
|
+
writeFileSync3(envFile, Object.entries(spec.env).map(([key, value]) => `${key}=${value.replace(/\n/g, "\\n")}`).join("\n"), { mode: 384 });
|
|
836
|
+
this.persistDeadline(running);
|
|
837
|
+
writeWorkerJson(join4(directory, "runtime.json"), { sessionId: spec.sessionId, image: spec.image, fingerprint });
|
|
838
|
+
} catch (error) {
|
|
839
|
+
rmSync2(directory, { recursive: true, force: true });
|
|
840
|
+
throw error;
|
|
841
|
+
}
|
|
842
|
+
this.current = running;
|
|
843
|
+
let accepted;
|
|
844
|
+
const ready = new Promise((resolve) => {
|
|
845
|
+
accepted = resolve;
|
|
846
|
+
});
|
|
847
|
+
const finished = () => this.finished(running);
|
|
848
|
+
running.completion = Promise.resolve().then(() => running.controller.signal.aborted ? void 0 : this.docker.run({
|
|
849
|
+
image: spec.image,
|
|
850
|
+
envFile,
|
|
851
|
+
port: this.port,
|
|
852
|
+
signal: running.controller.signal,
|
|
853
|
+
onReady: accepted,
|
|
854
|
+
...spec.lease ? { deadlineFile: join4(directory, "lease.json") } : {}
|
|
855
|
+
})).then(finished, (error) => {
|
|
856
|
+
if (error instanceof WorkerProcessError) finished();
|
|
857
|
+
throw error;
|
|
858
|
+
}).finally(() => {
|
|
859
|
+
clearTimeout(running.deadlineTimer);
|
|
860
|
+
rmSync2(envFile, { force: true });
|
|
861
|
+
});
|
|
862
|
+
void running.completion.catch(() => {
|
|
863
|
+
});
|
|
864
|
+
this.scheduleDeadline(running);
|
|
865
|
+
await Promise.race([ready, running.completion.then(() => {
|
|
866
|
+
throw new Error("Worker ended before accepting its container");
|
|
867
|
+
})]);
|
|
868
|
+
}
|
|
869
|
+
async renew(value) {
|
|
870
|
+
const next = parseWorkerLease(value), running = this.current;
|
|
871
|
+
if (!running?.spec.lease || running.stopping || !canRenewWorkerLease(running.spec.lease, next)) throw new Error("Worker lease cannot be renewed");
|
|
872
|
+
try {
|
|
873
|
+
if (this.docker.renewLease) await this.docker.renewLease(join4(running.directory, "lease.json"), next);
|
|
874
|
+
else throw new Error("Worker runner cannot enforce execution leases");
|
|
875
|
+
} catch (error) {
|
|
876
|
+
void this.stop(running.spec.lease).catch(() => {
|
|
877
|
+
});
|
|
878
|
+
throw error;
|
|
879
|
+
}
|
|
880
|
+
if (this.current !== running || running.stopping || Date.parse(next.expiresAt) <= Date.now()) throw new Error("Runtime ended before renewal acknowledgement");
|
|
881
|
+
running.spec = { ...running.spec, lease: next };
|
|
882
|
+
this.scheduleDeadline(running);
|
|
883
|
+
}
|
|
884
|
+
async stop(expected) {
|
|
885
|
+
const running = this.current;
|
|
886
|
+
if (!running) return;
|
|
887
|
+
if (expected && !running.spec.lease || running.spec.lease && (!expected || !sameWorkerLeaseIdentity(running.spec.lease, expected))) throw new Error("Worker stop targets another execution");
|
|
888
|
+
running.stopping = true;
|
|
889
|
+
clearTimeout(running.deadlineTimer);
|
|
890
|
+
running.controller.abort();
|
|
891
|
+
try {
|
|
892
|
+
await running.completion;
|
|
893
|
+
} catch (error) {
|
|
894
|
+
if (!(error instanceof WorkerProcessError)) throw error;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
async shutdown() {
|
|
898
|
+
await this.stop(this.current?.spec.lease);
|
|
899
|
+
}
|
|
900
|
+
async disconnected() {
|
|
901
|
+
if (!this.current?.spec.lease) await this.stop();
|
|
902
|
+
}
|
|
903
|
+
async completed() {
|
|
904
|
+
await this.current?.completion;
|
|
905
|
+
}
|
|
906
|
+
accepts(lease) {
|
|
907
|
+
if (!this.current) return false;
|
|
908
|
+
if (!this.current.spec.lease) return lease === void 0;
|
|
909
|
+
return !this.current.stopping && !!lease && sameWorkerLeaseIdentity(this.current.spec.lease, lease) && Date.parse(this.current.spec.lease.expiresAt) > Date.now();
|
|
910
|
+
}
|
|
911
|
+
finished(running) {
|
|
912
|
+
if (running.spec.lease) recordRetiredWorkerLease(this.stateDirectory, running.spec.lease);
|
|
913
|
+
if (this.current === running) this.current = null;
|
|
914
|
+
rmSync2(running.directory, { recursive: true, force: true });
|
|
915
|
+
}
|
|
916
|
+
persistDeadline(running) {
|
|
917
|
+
if (running.spec.lease) writeWorkerJson(join4(running.directory, "lease.json"), running.spec.lease);
|
|
918
|
+
}
|
|
919
|
+
scheduleDeadline(running) {
|
|
920
|
+
clearTimeout(running.deadlineTimer);
|
|
921
|
+
if (!running.spec.lease || this.docker.independentLease) return;
|
|
922
|
+
const expiresAt = running.spec.lease.expiresAt;
|
|
923
|
+
running.deadlineTimer = setTimeout(() => {
|
|
924
|
+
if (this.current !== running || running.spec.lease?.expiresAt !== expiresAt) return;
|
|
925
|
+
if (Date.parse(expiresAt) > Date.now()) {
|
|
926
|
+
this.scheduleDeadline(running);
|
|
927
|
+
return;
|
|
928
|
+
}
|
|
929
|
+
void this.stop(running.spec.lease).catch((error) => console.error("[sky worker] expired runtime cleanup remains unconfirmed", error));
|
|
930
|
+
}, Math.min(2147483647, Math.max(0, Date.parse(expiresAt) - Date.now())));
|
|
931
|
+
}
|
|
932
|
+
};
|
|
933
|
+
|
|
934
|
+
// src/worker.ts
|
|
935
|
+
var DEFAULT_POD_PORT = 18080;
|
|
936
|
+
var HEARTBEAT_INTERVAL_MS = 15e3;
|
|
937
|
+
var RECONNECT_MIN_MS = 250;
|
|
938
|
+
var RECONNECT_MAX_MS = 1e4;
|
|
939
|
+
async function startWorker(opts) {
|
|
940
|
+
const docker = opts.docker ?? new RealDockerRunner();
|
|
941
|
+
const signal = opts.signal ?? new AbortController().signal;
|
|
942
|
+
const podPort = opts.podPort ?? DEFAULT_POD_PORT;
|
|
943
|
+
const runtime = new WorkerRuntime(docker, podPort, opts.stateDirectory);
|
|
944
|
+
try {
|
|
945
|
+
let connectOnce2 = function() {
|
|
946
|
+
return new Promise((resolve) => {
|
|
947
|
+
const ws = new WS(connectUrl, { headers: { authorization: `Bearer ${opts.token}` } });
|
|
948
|
+
let epoch = 0;
|
|
949
|
+
let settled = false;
|
|
950
|
+
let controls = Promise.resolve();
|
|
951
|
+
const finish = (outcome, reason) => {
|
|
952
|
+
if (settled) return;
|
|
953
|
+
settled = true;
|
|
954
|
+
clearInterval(heartbeatTimer);
|
|
955
|
+
signal.removeEventListener("abort", onAbort);
|
|
956
|
+
resolve({ outcome, acknowledged: epoch > 0, reason });
|
|
957
|
+
};
|
|
958
|
+
const heartbeatTimer = setInterval(() => {
|
|
959
|
+
if (ws.readyState === WS.OPEN && epoch) ws.send(JSON.stringify({ type: "heartbeat" }));
|
|
960
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
961
|
+
const onAbort = () => {
|
|
962
|
+
ws.close(1e3, "Worker stopped");
|
|
963
|
+
finish("aborted");
|
|
964
|
+
};
|
|
965
|
+
if (signal.aborted) {
|
|
966
|
+
onAbort();
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
970
|
+
ws.on("open", () => {
|
|
971
|
+
ws.send(JSON.stringify({
|
|
972
|
+
type: "hello",
|
|
973
|
+
label: opts.label,
|
|
974
|
+
imageDigest: "unknown",
|
|
975
|
+
capacity: runtime.active ? 0 : 1,
|
|
976
|
+
capabilities: ["runtime-lease-v1", "connection-epoch-v1"],
|
|
977
|
+
...identity ? { workerId: identity.workerId, resumeToken: identity.resumeToken } : {},
|
|
978
|
+
...runtime.lease ? { lease: runtime.lease } : {}
|
|
979
|
+
}));
|
|
980
|
+
});
|
|
981
|
+
ws.on("message", (raw) => {
|
|
982
|
+
let msg;
|
|
983
|
+
try {
|
|
984
|
+
msg = JSON.parse(raw.toString());
|
|
985
|
+
} catch {
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
if (msg.type === "hello_ack") {
|
|
989
|
+
const next = acceptHelloAck2(msg);
|
|
990
|
+
if (!next) {
|
|
991
|
+
ws.close(1008, "Stale worker connection epoch");
|
|
992
|
+
finish("rejected", "stale or invalid connection epoch");
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
identity = next;
|
|
996
|
+
epoch = next.epoch;
|
|
997
|
+
console.log(`[sky worker] connected to pool ${opts.poolId} as ${next.workerId} (epoch ${next.epoch})`);
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
const current = epoch > 0 && msg.epoch === epoch;
|
|
1001
|
+
if (msg.type === "start" || msg.type === "renew" || msg.type === "stop") {
|
|
1002
|
+
controls = controls.then(async () => {
|
|
1003
|
+
if (!current) throw new Error("Worker control names a stale connection epoch");
|
|
1004
|
+
if (signal.aborted || ws.readyState !== WS.OPEN) throw new Error("Worker control channel is closed");
|
|
1005
|
+
const lease = msg.lease ? parseWorkerLease(msg.lease) : void 0;
|
|
1006
|
+
if (lease && (!msg.requestId || !/^[A-Za-z0-9_-]{1,128}$/.test(msg.requestId))) throw new Error("Lease control requires a request nonce");
|
|
1007
|
+
if (msg.type === "start") await runtime.start({ ...msg, lease });
|
|
1008
|
+
else if (msg.type === "renew") {
|
|
1009
|
+
if (!lease) throw new Error("Renewal requires an execution lease");
|
|
1010
|
+
await runtime.renew(lease);
|
|
1011
|
+
} else await runtime.stop(lease);
|
|
1012
|
+
if (msg.requestId && ws.readyState === WS.OPEN) ws.send(JSON.stringify({ type: "lease_ack", requestId: msg.requestId, action: msg.type, lease, status: msg.type === "stop" ? "stopped" : "accepted" }));
|
|
1013
|
+
}).catch((error) => {
|
|
1014
|
+
if (msg.requestId && ws.readyState === WS.OPEN) ws.send(JSON.stringify({ type: "lease_ack", requestId: msg.requestId, action: msg.type, lease: msg.lease, status: "rejected" }));
|
|
1015
|
+
else console.error("[sky worker] runtime control failed", error);
|
|
1016
|
+
});
|
|
1017
|
+
} else if (msg.type === "http") {
|
|
1018
|
+
if (!current || !runtime.accepts(msg.lease)) {
|
|
1019
|
+
ws.send(JSON.stringify({ type: "http_response", id: msg.id, status: current ? 403 : 409, headers: {}, body: "Runtime lease does not authorize this request" }));
|
|
1020
|
+
} else void handleHttp(msg, ws);
|
|
1021
|
+
} else if (msg.type === "stream_open") {
|
|
1022
|
+
if (!current || !runtime.accepts(msg.lease)) ws.send(JSON.stringify({ type: "stream_close", id: msg.id }));
|
|
1023
|
+
else void handleStreamOpen(msg, ws);
|
|
1024
|
+
}
|
|
1025
|
+
});
|
|
1026
|
+
ws.on("close", (code, reason) => {
|
|
1027
|
+
if (code === 1008) finish(/replaced/i.test(reason.toString()) ? "replaced" : "rejected", reason.toString() || "policy violation");
|
|
1028
|
+
else finish("transport");
|
|
1029
|
+
});
|
|
1030
|
+
ws.on("error", (err) => {
|
|
1031
|
+
const refused = /Unexpected server response: (4\d\d)/.exec(err.message);
|
|
1032
|
+
if (refused) finish("rejected", `HTTP ${refused[1]}`);
|
|
1033
|
+
else finish("transport");
|
|
1034
|
+
});
|
|
1035
|
+
});
|
|
1036
|
+
}, acceptHelloAck2 = function(msg) {
|
|
1037
|
+
if (typeof msg.workerId !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(msg.workerId) || !Number.isSafeInteger(msg.connectionEpoch) || msg.connectionEpoch < 1) return null;
|
|
1038
|
+
if (identity) {
|
|
1039
|
+
if (msg.workerId !== identity.workerId || msg.connectionEpoch <= identity.epoch) return null;
|
|
1040
|
+
if (msg.resumeToken !== void 0) return null;
|
|
1041
|
+
const next2 = { ...identity, epoch: msg.connectionEpoch };
|
|
1042
|
+
writeWorkerIdentity(runtime.root, next2);
|
|
1043
|
+
return next2;
|
|
1044
|
+
}
|
|
1045
|
+
if (runtime.active || typeof msg.resumeToken !== "string" || !/^wrt_[A-Za-z0-9_-]{32,128}$/.test(msg.resumeToken)) return null;
|
|
1046
|
+
const next = { poolId: opts.poolId, workerId: msg.workerId, resumeToken: msg.resumeToken, epoch: msg.connectionEpoch };
|
|
1047
|
+
writeWorkerIdentity(runtime.root, next);
|
|
1048
|
+
return next;
|
|
1049
|
+
};
|
|
1050
|
+
var connectOnce = connectOnce2, acceptHelloAck = acceptHelloAck2;
|
|
1051
|
+
await runtime.recover();
|
|
1052
|
+
let identity = readWorkerIdentity(runtime.root, opts.poolId);
|
|
1053
|
+
if (runtime.active && !identity) throw new Error("A retained worker execution has no resume identity; refusing to advertise capacity");
|
|
1054
|
+
const wsUrl = opts.apiBase.replace(/^https?:\/\//, (m) => m === "https://" ? "wss://" : "ws://").replace(/\/$/, "");
|
|
1055
|
+
const connectUrl = `${wsUrl}/api/v1/worker-pools/${opts.poolId}/connect`;
|
|
1056
|
+
const { default: WS } = await import("ws");
|
|
1057
|
+
let delay = RECONNECT_MIN_MS;
|
|
1058
|
+
for (; ; ) {
|
|
1059
|
+
const result = await connectOnce2();
|
|
1060
|
+
if (result.outcome === "aborted" || signal.aborted) {
|
|
1061
|
+
await runtime.shutdown();
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
await runtime.disconnected();
|
|
1065
|
+
if (result.outcome !== "transport") {
|
|
1066
|
+
await runtime.completed();
|
|
1067
|
+
throw new Error(result.outcome === "replaced" ? "Another process resumed this worker identity" : `Worker connection rejected: ${result.reason}`);
|
|
1068
|
+
}
|
|
1069
|
+
delay = result.acknowledged ? RECONNECT_MIN_MS : Math.min(RECONNECT_MAX_MS, delay * 2);
|
|
1070
|
+
await new Promise((resolve) => {
|
|
1071
|
+
const timer = setTimeout(done, delay);
|
|
1072
|
+
function done() {
|
|
1073
|
+
clearTimeout(timer);
|
|
1074
|
+
signal.removeEventListener("abort", done);
|
|
1075
|
+
resolve();
|
|
1076
|
+
}
|
|
1077
|
+
signal.addEventListener("abort", done, { once: true });
|
|
1078
|
+
});
|
|
1079
|
+
if (signal.aborted) {
|
|
1080
|
+
await runtime.shutdown();
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
} finally {
|
|
1085
|
+
runtime.release();
|
|
1086
|
+
}
|
|
1087
|
+
async function handleHttp(msg, ws) {
|
|
1088
|
+
const url = `http://127.0.0.1:${podPort}${msg.path}`;
|
|
1089
|
+
try {
|
|
1090
|
+
const response = await fetch(url, {
|
|
1091
|
+
method: msg.method,
|
|
1092
|
+
headers: msg.headers,
|
|
1093
|
+
body: msg.body || void 0,
|
|
1094
|
+
signal: AbortSignal.timeout(3e4)
|
|
1095
|
+
});
|
|
1096
|
+
const body = await response.text();
|
|
1097
|
+
const headers = {};
|
|
1098
|
+
response.headers.forEach((v, k) => {
|
|
1099
|
+
headers[k] = v;
|
|
1100
|
+
});
|
|
1101
|
+
const resp = {
|
|
1102
|
+
type: "http_response",
|
|
1103
|
+
id: msg.id,
|
|
1104
|
+
status: response.status,
|
|
1105
|
+
headers,
|
|
1106
|
+
body
|
|
1107
|
+
};
|
|
1108
|
+
ws.send(JSON.stringify(resp));
|
|
1109
|
+
} catch (err) {
|
|
1110
|
+
ws.send(JSON.stringify({
|
|
1111
|
+
type: "http_response",
|
|
1112
|
+
id: msg.id,
|
|
1113
|
+
status: 502,
|
|
1114
|
+
headers: {},
|
|
1115
|
+
body: String(err)
|
|
1116
|
+
}));
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
async function handleStreamOpen(msg, ws) {
|
|
1120
|
+
const url = `http://127.0.0.1:${podPort}${msg.path}`;
|
|
1121
|
+
try {
|
|
1122
|
+
const response = await fetch(url, {
|
|
1123
|
+
headers: { accept: "text/event-stream" },
|
|
1124
|
+
signal: AbortSignal.timeout(3e5)
|
|
1125
|
+
});
|
|
1126
|
+
const reader = response.body?.getReader();
|
|
1127
|
+
if (!reader) {
|
|
1128
|
+
ws.send(JSON.stringify({ type: "stream_close", id: msg.id }));
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
const decoder = new TextDecoder();
|
|
1132
|
+
for (; ; ) {
|
|
1133
|
+
const { done, value } = await reader.read();
|
|
1134
|
+
if (done) break;
|
|
1135
|
+
ws.send(JSON.stringify({
|
|
1136
|
+
type: "stream_event",
|
|
1137
|
+
id: msg.id,
|
|
1138
|
+
data: decoder.decode(value, { stream: true })
|
|
1139
|
+
}));
|
|
1140
|
+
}
|
|
1141
|
+
ws.send(JSON.stringify({ type: "stream_close", id: msg.id }));
|
|
1142
|
+
} catch {
|
|
1143
|
+
ws.send(JSON.stringify({ type: "stream_close", id: msg.id }));
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
export {
|
|
1148
|
+
RealDockerRunner,
|
|
1149
|
+
startWorker
|
|
1150
|
+
};
|