@jameslovespancakes/pi-plus 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +190 -190
  3. package/config/pi-plus.example.json +60 -60
  4. package/config/skills/model-routing/SKILL.md +86 -86
  5. package/images/pi-plus.svg +10 -10
  6. package/package.json +67 -67
  7. package/server/board-server.mjs +641 -641
  8. package/server/package.json +17 -17
  9. package/src/core/accounts/registry.ts +93 -93
  10. package/src/core/anthropic/client-identity.ts +241 -241
  11. package/src/core/catalog/quality.ts +314 -314
  12. package/src/core/config.ts +169 -169
  13. package/src/core/env.ts +58 -58
  14. package/src/core/exec/process.ts +146 -146
  15. package/src/core/exec/ssh-config.ts +157 -157
  16. package/src/core/policy/policy.ts +183 -183
  17. package/src/core/quota/pool.ts +64 -64
  18. package/src/core/quota/usage-source.ts +289 -289
  19. package/src/core/store.ts +43 -43
  20. package/src/domains/agents/board-setup.ts +409 -409
  21. package/src/domains/agents/index.ts +462 -462
  22. package/src/domains/models/catalog-tool.ts +361 -361
  23. package/src/domains/models/index.ts +14 -14
  24. package/src/domains/models/policy-gate.ts +169 -169
  25. package/src/domains/models/provider-picker.ts +207 -207
  26. package/src/domains/remote/config-path.ts +41 -41
  27. package/src/domains/remote/index.ts +866 -866
  28. package/src/domains/remote/setup.ts +425 -425
  29. package/src/domains/setup/index.ts +220 -220
  30. package/src/domains/subscriptions/accounts.ts +242 -242
  31. package/src/domains/subscriptions/footer.ts +182 -182
  32. package/src/domains/subscriptions/index.ts +42 -42
  33. package/src/domains/subscriptions/provider.ts +219 -219
  34. package/src/domains/subscriptions/providers/anthropic.ts +149 -149
  35. package/src/domains/subscriptions/providers/codex.ts +148 -148
  36. package/src/domains/subscriptions/routing.ts +72 -72
  37. package/src/services/usage-service.ts +186 -186
  38. package/src/ui/format.ts +73 -73
  39. package/src/ui/usage-bars.ts +154 -154
  40. package/src/vendor/anthropic.ts +109 -109
@@ -1,866 +1,866 @@
1
- import { createHash, randomBytes } from "node:crypto";
2
- import { createReadStream } from "node:fs";
3
- import { lstat, mkdtemp, readlink, rm, writeFile } from "node:fs/promises";
4
- import { basename, isAbsolute, relative, resolve, sep } from "node:path";
5
- import { homedir, tmpdir } from "node:os";
6
- import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
7
- import {
8
- DEFAULT_MAX_BYTES,
9
- DEFAULT_MAX_LINES,
10
- formatSize,
11
- truncateTail,
12
- } from "@earendil-works/pi-coding-agent";
13
- import { StringEnum } from "@earendil-works/pi-ai";
14
- import { registerRemoteSetup } from "./setup.ts";
15
- import { readRemote } from "./config-path.ts";
16
- import { Type } from "typebox";
17
- import {
18
- appendTail,
19
- shQuote,
20
- rootAssignment,
21
- runLocal,
22
- runProcess,
23
- runSshCommand,
24
- type ProcessResult,
25
- type RunOptions,
26
- } from "../../core/exec/process.ts";
27
-
28
- interface Limits {
29
- cpuBlockPercent: number;
30
- gpuBlockPercent: number;
31
- gpuMemoryBlockPercent: number;
32
- memoryBlockPercent: number;
33
- minimumFreeDiskGB: number;
34
- statusCacheSeconds: number;
35
- retentionHours: number;
36
- }
37
-
38
- interface WorkerInput extends Partial<Limits> {
39
- name: string;
40
- ssh: string;
41
- root?: string;
42
- nice?: number;
43
- tags?: string[];
44
- /** Explicit key path, for hosts added without an ~/.ssh/config entry. */
45
- identityFile?: string;
46
- port?: number;
47
- /** Toggled off in the picker without losing the entry. */
48
- enabled?: boolean;
49
- }
50
-
51
- // WorkerInput carries the same limit keys as optional overrides, so they must be
52
- // stripped before re-declaring them as required or the two bases conflict.
53
- interface Worker extends Omit<WorkerInput, keyof Limits | "enabled">, Limits {
54
- enabled: boolean;
55
- root: string;
56
- nice: number;
57
- tags: string[];
58
- }
59
-
60
- interface Config {
61
- injectStatus: boolean;
62
- workers: Worker[];
63
- }
64
-
65
- interface WorkerStatus {
66
- name: string;
67
- ssh: string;
68
- os?: string;
69
- state: "ready" | "blocked" | "unreachable";
70
- cpuPercent?: number;
71
- memoryPercent?: number;
72
- gpuPercent?: number;
73
- gpuMemoryPercent?: number;
74
- freeDiskGB?: number;
75
- jobs?: number;
76
- gpuJobs?: number;
77
- thermal?: string;
78
- modelMode?: boolean;
79
- reasons: string[];
80
- tags: string[];
81
- sampledAt: string;
82
- }
83
-
84
- interface SnapshotResult {
85
- tempDir: string;
86
- archivePath: string;
87
- repoRoot: string;
88
- repoName: string;
89
- mode: "working-tree" | "tracked" | "paths";
90
- files: string[];
91
- fingerprint: string;
92
- archiveBytes: number;
93
- commit: string;
94
- dirty: boolean;
95
- }
96
-
97
- const DEFAULT_LIMITS: Limits = {
98
- cpuBlockPercent: 90,
99
- gpuBlockPercent: 90,
100
- gpuMemoryBlockPercent: 90,
101
- memoryBlockPercent: 90,
102
- minimumFreeDiskGB: 10,
103
- statusCacheSeconds: 30,
104
- retentionHours: 24,
105
- };
106
-
107
- const HARD_WALK_EXCLUDES = new Set([
108
- ".git",
109
- "node_modules",
110
- ".venv",
111
- "__pycache__",
112
- "target",
113
- "dist",
114
- "build",
115
- "coverage",
116
- ".next",
117
- "remote_tests",
118
- ]);
119
-
120
- let statusCache: { expiresAt: number; key: string; statuses: WorkerStatus[] } | undefined;
121
-
122
- function finiteNumber(value: unknown, fallback: number, min: number, max: number): number {
123
- return typeof value === "number" && Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback;
124
- }
125
-
126
- async function loadConfig(): Promise<Config> {
127
- // The `remote` section of pi-plus.json. An empty worker list is the normal
128
- // first-run state: /remote setup populates it.
129
- const parsed = readRemote();
130
- const defaults = parsed.defaults ?? {};
131
- const defaultLimits: Limits = {
132
- cpuBlockPercent: finiteNumber(defaults.cpuBlockPercent, DEFAULT_LIMITS.cpuBlockPercent, 1, 100),
133
- gpuBlockPercent: finiteNumber(defaults.gpuBlockPercent, DEFAULT_LIMITS.gpuBlockPercent, 1, 100),
134
- gpuMemoryBlockPercent: finiteNumber(
135
- defaults.gpuMemoryBlockPercent,
136
- DEFAULT_LIMITS.gpuMemoryBlockPercent,
137
- 1,
138
- 100,
139
- ),
140
- memoryBlockPercent: finiteNumber(defaults.memoryBlockPercent, DEFAULT_LIMITS.memoryBlockPercent, 1, 100),
141
- minimumFreeDiskGB: finiteNumber(defaults.minimumFreeDiskGB, DEFAULT_LIMITS.minimumFreeDiskGB, 0, 100000),
142
- statusCacheSeconds: finiteNumber(defaults.statusCacheSeconds, DEFAULT_LIMITS.statusCacheSeconds, 1, 600),
143
- retentionHours: finiteNumber(defaults.retentionHours, DEFAULT_LIMITS.retentionHours, 1, 24 * 365),
144
- };
145
- const names = new Set<string>();
146
- const workers: Worker[] = parsed.workers.map((raw: WorkerInput, index: number) => {
147
- if (!raw || typeof raw.name !== "string" || !raw.name.trim() || typeof raw.ssh !== "string" || !raw.ssh.trim()) {
148
- throw new Error(`Invalid worker at index ${index}: name and ssh are required strings.`);
149
- }
150
- if (names.has(raw.name)) throw new Error(`Duplicate remote worker name: ${raw.name}`);
151
- names.add(raw.name);
152
- const root = typeof raw.root === "string" && raw.root.trim() ? raw.root.trim() : "~/remote_tests";
153
- if (root.includes("\n") || root.includes("\0") || (!root.startsWith("~/") && !root.startsWith("/"))) {
154
- throw new Error(`Worker ${raw.name} root must be an absolute POSIX path or start with ~/`);
155
- }
156
- return {
157
- name: raw.name.trim(),
158
- ssh: raw.ssh.trim(),
159
- root,
160
- enabled: raw.enabled !== false,
161
- identityFile: typeof raw.identityFile === "string" && raw.identityFile.trim() ? raw.identityFile.trim() : undefined,
162
- port: Number.isInteger(raw.port) && raw.port! > 0 && raw.port! < 65536 ? raw.port : undefined,
163
- nice: Math.floor(finiteNumber(raw.nice, 10, 0, 19)),
164
- tags: Array.isArray(raw.tags) ? raw.tags.filter((tag): tag is string => typeof tag === "string") : [],
165
- cpuBlockPercent: finiteNumber(raw.cpuBlockPercent, defaultLimits.cpuBlockPercent, 1, 100),
166
- gpuBlockPercent: finiteNumber(raw.gpuBlockPercent, defaultLimits.gpuBlockPercent, 1, 100),
167
- gpuMemoryBlockPercent: finiteNumber(
168
- raw.gpuMemoryBlockPercent,
169
- defaultLimits.gpuMemoryBlockPercent,
170
- 1,
171
- 100,
172
- ),
173
- memoryBlockPercent: finiteNumber(raw.memoryBlockPercent, defaultLimits.memoryBlockPercent, 1, 100),
174
- minimumFreeDiskGB: finiteNumber(raw.minimumFreeDiskGB, defaultLimits.minimumFreeDiskGB, 0, 100000),
175
- statusCacheSeconds: finiteNumber(raw.statusCacheSeconds, defaultLimits.statusCacheSeconds, 1, 600),
176
- retentionHours: finiteNumber(raw.retentionHours, defaultLimits.retentionHours, 1, 24 * 365),
177
- };
178
- });
179
- return { injectStatus: parsed.injectStatus !== false, workers };
180
- }
181
-
182
- /** `-i`/`-p` only when the worker was added without an ~/.ssh/config entry. */
183
- function sshArgsFor(worker: Worker): string[] {
184
- const args: string[] = [];
185
- if (worker.identityFile) {
186
- const expanded = worker.identityFile.startsWith("~/")
187
- ? resolve(homedir(), worker.identityFile.slice(2))
188
- : worker.identityFile;
189
- args.push("-i", expanded, "-o", "IdentitiesOnly=yes");
190
- }
191
- if (worker.port) args.push("-p", String(worker.port));
192
- return args;
193
- }
194
-
195
- async function runSsh(
196
- worker: Worker,
197
- remoteCommand: string,
198
- options: RunOptions = {},
199
- ): Promise<ProcessResult> {
200
- return runSshCommand(worker.ssh, remoteCommand, options, sshArgsFor(worker));
201
- }
202
-
203
- function healthScript(worker: Worker): string {
204
- return `
205
- set +e
206
- LC_ALL=C
207
- ${rootAssignment(worker.root)}
208
- OS=$(uname -s 2>/dev/null || echo unknown)
209
- CPU=-1
210
- MEM=-1
211
- GPU=-1
212
- GPU_MEM=-1
213
- THERMAL=normal
214
- if [ "$OS" = Darwin ]; then
215
- CORES=$(sysctl -n hw.logicalcpu 2>/dev/null || echo 1)
216
- CPU=$(ps -A -o %cpu= 2>/dev/null | awk -v c="$CORES" '{s+=$1} END {if(c<1)c=1; v=s/c; if(v>100)v=100; printf "%.1f",v}')
217
- FREE=$(memory_pressure -Q 2>/dev/null | awk -F': ' '/System-wide memory free percentage/ {gsub(/%/,"",$2); print $2; exit}')
218
- if [ -n "$FREE" ]; then MEM=$(awk -v f="$FREE" 'BEGIN {printf "%.1f",100-f}'); fi
219
- if ! pmset -g therm 2>/dev/null | grep -q 'No thermal warning level'; then THERMAL=warning; fi
220
- else
221
- set -- $(awk '/^cpu / {idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i; print total,idle; exit}' /proc/stat 2>/dev/null)
222
- T1=$1; I1=$2
223
- sleep 0.4
224
- set -- $(awk '/^cpu / {idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i; print total,idle; exit}' /proc/stat 2>/dev/null)
225
- T2=$1; I2=$2
226
- if [ -n "$T1" ] && [ "$T2" -gt "$T1" ] 2>/dev/null; then CPU=$(awk -v t1="$T1" -v i1="$I1" -v t2="$T2" -v i2="$I2" 'BEGIN {printf "%.1f",100*(1-(i2-i1)/(t2-t1))}'); fi
227
- MEM=$(awk '/MemTotal/ {t=$2} /MemAvailable/ {a=$2} END {if(t>0) printf "%.1f",100*(t-a)/t; else print -1}' /proc/meminfo 2>/dev/null)
228
- if command -v nvidia-smi >/dev/null 2>&1; then
229
- GPU_LINE=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')
230
- GPU=$(printf '%s' "$GPU_LINE" | cut -d, -f1)
231
- GPU_USED=$(printf '%s' "$GPU_LINE" | cut -d, -f2)
232
- GPU_TOTAL=$(printf '%s' "$GPU_LINE" | cut -d, -f3)
233
- if [ -n "$GPU_TOTAL" ] && [ "$GPU_TOTAL" -gt 0 ] 2>/dev/null; then GPU_MEM=$(awk -v u="$GPU_USED" -v t="$GPU_TOTAL" 'BEGIN {printf "%.1f",100*u/t}'); fi
234
- fi
235
- fi
236
- FREE_KB=$(df -Pk "$HOME" 2>/dev/null | awk 'NR==2 {print $4}')
237
- FREE_GB=$(awk -v k="\${FREE_KB:-0}" 'BEGIN {printf "%.1f",k/1048576}')
238
- JOBS=0
239
- GPU_JOBS=0
240
- for d in "$ROOT/.slots"/*; do
241
- [ -d "$d" ] || continue
242
- JOBS=$((JOBS+1))
243
- [ -f "$d/gpu" ] && GPU_JOBS=$((GPU_JOBS+1))
244
- done
245
- MODEL_MODE=0
246
- if [ -e "$ROOT/.model-mode" ] || [ -e "$HOME/.model-mode" ]; then MODEL_MODE=1; fi
247
- printf 'OS=%s\nCPU=%s\nMEM=%s\nGPU=%s\nGPU_MEM=%s\nFREE_GB=%s\nJOBS=%s\nGPU_JOBS=%s\nTHERMAL=%s\nMODEL_MODE=%s\n' "$OS" "$CPU" "$MEM" "$GPU" "$GPU_MEM" "$FREE_GB" "$JOBS" "$GPU_JOBS" "$THERMAL" "$MODEL_MODE"
248
- `;
249
- }
250
-
251
- function parseNumber(value: string | undefined): number | undefined {
252
- if (value === undefined) return undefined;
253
- const parsed = Number(value);
254
- return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
255
- }
256
-
257
- async function probeWorker(worker: Worker): Promise<WorkerStatus> {
258
- const sampledAt = new Date().toISOString();
259
- try {
260
- const result = await runSsh(worker, "bash -s", { input: healthScript(worker), timeoutSeconds: 12 });
261
- if (result.code !== 0 || result.timedOut || result.aborted) {
262
- const message = result.timedOut ? "health check timed out" : (result.stderr.trim() || `SSH exited ${result.code}`);
263
- return {
264
- name: worker.name,
265
- ssh: worker.ssh,
266
- state: "unreachable",
267
- reasons: [message],
268
- tags: worker.tags,
269
- sampledAt,
270
- };
271
- }
272
- const values = new Map<string, string>();
273
- for (const line of result.stdout.split(/\r?\n/)) {
274
- const separator = line.indexOf("=");
275
- if (separator > 0) values.set(line.slice(0, separator), line.slice(separator + 1));
276
- }
277
- const cpuPercent = parseNumber(values.get("CPU"));
278
- const memoryPercent = parseNumber(values.get("MEM"));
279
- const gpuPercent = parseNumber(values.get("GPU"));
280
- const gpuMemoryPercent = parseNumber(values.get("GPU_MEM"));
281
- const freeDiskGB = parseNumber(values.get("FREE_GB"));
282
- const jobs = parseNumber(values.get("JOBS")) ?? 0;
283
- const gpuJobs = parseNumber(values.get("GPU_JOBS")) ?? 0;
284
- const thermal = values.get("THERMAL") || "unknown";
285
- const modelMode = values.get("MODEL_MODE") === "1";
286
- const reasons: string[] = [];
287
- if (modelMode) reasons.push("model mode is active");
288
- if (cpuPercent !== undefined && cpuPercent >= worker.cpuBlockPercent) {
289
- reasons.push(`CPU ${cpuPercent.toFixed(1)}% >= ${worker.cpuBlockPercent}%`);
290
- }
291
- if (memoryPercent !== undefined && memoryPercent >= worker.memoryBlockPercent) {
292
- reasons.push(`memory ${memoryPercent.toFixed(1)}% >= ${worker.memoryBlockPercent}%`);
293
- }
294
- if (gpuPercent !== undefined && gpuPercent >= worker.gpuBlockPercent) {
295
- reasons.push(`GPU ${gpuPercent.toFixed(1)}% >= ${worker.gpuBlockPercent}%`);
296
- }
297
- if (gpuMemoryPercent !== undefined && gpuMemoryPercent >= worker.gpuMemoryBlockPercent) {
298
- reasons.push(`GPU memory ${gpuMemoryPercent.toFixed(1)}% >= ${worker.gpuMemoryBlockPercent}%`);
299
- }
300
- if (freeDiskGB !== undefined && freeDiskGB < worker.minimumFreeDiskGB) {
301
- reasons.push(`free disk ${freeDiskGB.toFixed(1)} GB < ${worker.minimumFreeDiskGB} GB`);
302
- }
303
- if (thermal !== "normal") reasons.push(`thermal state ${thermal}`);
304
- return {
305
- name: worker.name,
306
- ssh: worker.ssh,
307
- os: values.get("OS"),
308
- state: reasons.length ? "blocked" : "ready",
309
- cpuPercent,
310
- memoryPercent,
311
- gpuPercent,
312
- gpuMemoryPercent,
313
- freeDiskGB,
314
- jobs,
315
- gpuJobs,
316
- thermal,
317
- modelMode,
318
- reasons,
319
- tags: worker.tags,
320
- sampledAt,
321
- };
322
- } catch (error) {
323
- return {
324
- name: worker.name,
325
- ssh: worker.ssh,
326
- state: "unreachable",
327
- reasons: [error instanceof Error ? error.message : String(error)],
328
- tags: worker.tags,
329
- sampledAt,
330
- };
331
- }
332
- }
333
-
334
- async function probeWorkers(config: Config, force = false): Promise<WorkerStatus[]> {
335
- const active = config.workers.filter((worker) => worker.enabled);
336
- if (active.length === 0) return [];
337
- const key = JSON.stringify(active.map((worker) => [worker.name, worker.ssh, worker.root]));
338
- const ttlSeconds = Math.min(...active.map((worker) => worker.statusCacheSeconds));
339
- if (!force && statusCache && statusCache.key === key && statusCache.expiresAt > Date.now()) return statusCache.statuses;
340
- const statuses = await Promise.all(active.map(probeWorker));
341
- statusCache = { key, statuses, expiresAt: Date.now() + ttlSeconds * 1000 };
342
- return statuses;
343
- }
344
-
345
- function metric(value: number | undefined): string {
346
- return value === undefined ? "?" : `${Math.round(value)}%`;
347
- }
348
-
349
- function compactStatus(status: WorkerStatus): string {
350
- const gpu = status.gpuPercent === undefined ? "GPU ?" : `GPU ${metric(status.gpuPercent)}`;
351
- const reason = status.reasons.length ? ` (${status.reasons.join(", ")})` : "";
352
- return `${status.name} ${status.state.toUpperCase()} CPU ${metric(status.cpuPercent)} MEM ${metric(status.memoryPercent)} ${gpu} active-jobs ${status.jobs ?? "?"}${reason}`;
353
- }
354
-
355
- function detailedStatuses(statuses: WorkerStatus[]): string {
356
- return statuses
357
- .map((status) => {
358
- const fields = [
359
- `${status.name.padEnd(10)} ${status.state.toUpperCase().padEnd(11)}`,
360
- `CPU ${metric(status.cpuPercent).padStart(4)}`,
361
- `MEM ${metric(status.memoryPercent).padStart(4)}`,
362
- `GPU ${metric(status.gpuPercent).padStart(4)}`,
363
- `VRAM ${metric(status.gpuMemoryPercent).padStart(4)}`,
364
- `disk ${status.freeDiskGB === undefined ? "?" : `${status.freeDiskGB.toFixed(1)}GB`}`,
365
- `active-jobs ${status.jobs ?? "?"}`,
366
- ];
367
- return fields.join(" ") + (status.reasons.length ? `\n blocked: ${status.reasons.join("; ")}` : "");
368
- })
369
- .join("\n");
370
- }
371
-
372
- function normalizeRepoPath(input: string): string {
373
- const value = input.replace(/\\/g, "/").replace(/^\.\//, "").replace(/\/+$/, "");
374
- if (value === "." || value === "") return "";
375
- if (value.includes("\0") || value.includes("\n") || isAbsolute(value) || /^[A-Za-z]:/.test(value)) {
376
- throw new Error(`Snapshot path must be repository-relative: ${input}`);
377
- }
378
- const parts = value.split("/");
379
- if (parts.some((part) => part === "..")) throw new Error(`Snapshot path escapes the repository: ${input}`);
380
- if (parts.some((part) => part === ".git")) throw new Error(`Snapshot paths cannot include .git: ${input}`);
381
- return parts.filter((part) => part && part !== ".").join("/");
382
- }
383
-
384
- function matchesPath(file: string, selected: string): boolean {
385
- return selected === "" || file === selected || file.startsWith(`${selected}/`);
386
- }
387
-
388
- async function walkFiles(root: string, current = ""): Promise<string[]> {
389
- const directory = resolve(root, current || ".");
390
- const entries = await import("node:fs/promises").then(({ readdir }) => readdir(directory, { withFileTypes: true }));
391
- const files: string[] = [];
392
- for (const entry of entries) {
393
- if (HARD_WALK_EXCLUDES.has(entry.name)) continue;
394
- const rel = current ? `${current}/${entry.name}` : entry.name;
395
- if (entry.isDirectory()) files.push(...(await walkFiles(root, rel)));
396
- else if (entry.isFile() || entry.isSymbolicLink()) files.push(rel);
397
- }
398
- return files;
399
- }
400
-
401
- async function gitOutput(repoRoot: string, args: string[]): Promise<string> {
402
- const result = await runLocal("git", ["-C", repoRoot, ...args]);
403
- if (result.code !== 0) throw new Error(result.stderr.trim() || `git ${args.join(" ")} failed`);
404
- return result.stdout;
405
- }
406
-
407
- async function resolveRepository(repositoryPath: string | undefined, cwd: string): Promise<string> {
408
- const candidate = resolve(cwd, repositoryPath?.replace(/^@/, "") || ".");
409
- const result = await runLocal("git", ["-C", candidate, "rev-parse", "--show-toplevel"]);
410
- if (result.code !== 0) throw new Error(`${candidate} is not inside a Git repository.`);
411
- return resolve(result.stdout.trim());
412
- }
413
-
414
- async function selectFiles(
415
- repoRoot: string,
416
- mode: "working-tree" | "tracked" | "paths",
417
- paths: string[],
418
- excludes: string[],
419
- includeIgnored: boolean,
420
- ): Promise<string[]> {
421
- let files: string[];
422
- if (includeIgnored && mode !== "tracked") {
423
- files = await walkFiles(repoRoot);
424
- } else {
425
- const args = mode === "tracked" ? ["ls-files", "-z", "--cached"] : ["ls-files", "-z", "--cached", "--others", "--exclude-standard"];
426
- files = (await gitOutput(repoRoot, args)).split("\0").filter(Boolean).map((file) => file.replace(/\\/g, "/"));
427
- }
428
- const normalizedPaths = paths.map(normalizeRepoPath);
429
- const normalizedExcludes = excludes.map(normalizeRepoPath);
430
- if (mode === "paths") {
431
- if (normalizedPaths.length === 0) throw new Error('snapshot.mode "paths" requires at least one path.');
432
- files = files.filter((file) => normalizedPaths.some((selected) => matchesPath(file, selected)));
433
- }
434
- files = files.filter((file) => !normalizedExcludes.some((excluded) => matchesPath(file, excluded)));
435
- const existing: string[] = [];
436
- for (const file of [...new Set(files)].sort()) {
437
- const absolute = resolve(repoRoot, file.split("/").join(sep));
438
- if (relative(repoRoot, absolute).startsWith("..")) continue;
439
- try {
440
- const stat = await lstat(absolute);
441
- if (stat.isFile() || stat.isSymbolicLink()) existing.push(file);
442
- } catch {
443
- // Deleted tracked files are intentionally absent from the snapshot.
444
- }
445
- }
446
- if (existing.length === 0) throw new Error("Snapshot selection contains no files.");
447
- return existing;
448
- }
449
-
450
- async function fileStamp(repoRoot: string, files: string[]): Promise<string> {
451
- const hash = createHash("sha256");
452
- for (const file of files) {
453
- const absolute = resolve(repoRoot, file.split("/").join(sep));
454
- const stat = await lstat(absolute);
455
- hash.update(file).update("\0").update(`${stat.size}:${stat.mtimeMs}:${stat.mode}`).update("\0");
456
- if (stat.isSymbolicLink()) hash.update(await readlink(absolute));
457
- }
458
- return hash.digest("hex");
459
- }
460
-
461
- async function sha256File(path: string): Promise<string> {
462
- return new Promise((resolvePromise, reject) => {
463
- const hash = createHash("sha256");
464
- const stream = createReadStream(path);
465
- stream.on("data", (chunk) => hash.update(chunk));
466
- stream.on("error", reject);
467
- stream.on("end", () => resolvePromise(hash.digest("hex")));
468
- });
469
- }
470
-
471
- async function createSnapshot(
472
- cwd: string,
473
- params: {
474
- repositoryPath?: string;
475
- snapshot?: {
476
- mode?: "working-tree" | "tracked" | "paths";
477
- paths?: string[];
478
- excludePaths?: string[];
479
- includeIgnored?: boolean;
480
- };
481
- },
482
- ): Promise<SnapshotResult> {
483
- const repoRoot = await resolveRepository(params.repositoryPath, cwd);
484
- const mode = params.snapshot?.mode ?? "working-tree";
485
- const paths = params.snapshot?.paths ?? [];
486
- const excludes = params.snapshot?.excludePaths ?? [];
487
- const includeIgnored = params.snapshot?.includeIgnored ?? false;
488
- const repoName = basename(repoRoot).replace(/[^A-Za-z0-9._-]+/g, "-") || "repo";
489
- const tempDir = await mkdtemp(resolve(tmpdir(), "pi-remote-test-"));
490
- const archivePath = resolve(tempDir, "snapshot.tar.gz");
491
- try {
492
- for (let attempt = 1; attempt <= 2; attempt++) {
493
- const files = await selectFiles(repoRoot, mode, paths, excludes, includeIgnored);
494
- const before = await fileStamp(repoRoot, files);
495
- const listPath = resolve(tempDir, "files.nul");
496
- await writeFile(listPath, Buffer.from(`${files.join("\0")}\0`, "utf8"), { mode: 0o600 });
497
- const tar = await runProcess("tar", ["-czf", archivePath, "-C", repoRoot, "--no-recursion", "--null", "-T", listPath], {
498
- timeoutSeconds: 300,
499
- });
500
- if (tar.code !== 0) throw new Error(tar.stderr.trim() || "Failed to create snapshot archive.");
501
- const after = await fileStamp(repoRoot, files);
502
- if (before !== after) {
503
- if (attempt === 2) throw new Error("Selected files changed while the snapshot was being created; retry after edits settle.");
504
- continue;
505
- }
506
- const stat = await lstat(archivePath);
507
- const commit = (await gitOutput(repoRoot, ["rev-parse", "HEAD"])).trim();
508
- const dirty = (await gitOutput(repoRoot, ["status", "--porcelain", "--untracked-files=normal"])).length > 0;
509
- return {
510
- tempDir,
511
- archivePath,
512
- repoRoot,
513
- repoName,
514
- mode,
515
- files,
516
- fingerprint: await sha256File(archivePath),
517
- archiveBytes: stat.size,
518
- commit,
519
- dirty,
520
- };
521
- }
522
- throw new Error("Snapshot creation failed.");
523
- } catch (error) {
524
- await rm(tempDir, { recursive: true, force: true });
525
- throw error;
526
- }
527
- }
528
-
529
- function gpuCapable(worker: Worker): boolean {
530
- return worker.tags.some((tag) => ["gpu", "cuda", "metal"].includes(tag.toLowerCase()));
531
- }
532
-
533
- function pickWorker(config: Config, statuses: WorkerStatus[], requested: string, requiresGpu: boolean): Worker | undefined {
534
- if (requested !== "auto") return config.workers.find((worker) => worker.enabled && worker.name === requested);
535
- const ready = statuses
536
- .filter((status) => status.state === "ready")
537
- .map((status) => ({ status, worker: config.workers.find((worker) => worker.name === status.name)! }))
538
- .filter(({ worker }) => worker && (!requiresGpu || gpuCapable(worker)));
539
- ready.sort((a, b) => {
540
- const score = (item: (typeof ready)[number]) =>
541
- Math.max(item.status.cpuPercent ?? 0, item.status.memoryPercent ?? 0, item.status.gpuPercent ?? 0);
542
- return score(a) - score(b);
543
- });
544
- return ready[0]?.worker;
545
- }
546
-
547
- function reserveScript(worker: Worker, jobId: string, requiresGpu: boolean): string {
548
- return `
549
- set -u
550
- ${rootAssignment(worker.root)}
551
- mkdir -p "$ROOT/.slots"
552
- LOCK="$ROOT/.admission.lock"
553
- NOW=$(date +%s)
554
- for d in "$ROOT/.slots"/*; do
555
- [ -d "$d" ] || continue
556
- CREATED=$(cat "$d/created" 2>/dev/null || echo "$NOW")
557
- if [ $((NOW-CREATED)) -gt 21600 ]; then rm -rf "$d"; fi
558
- done
559
- ACQUIRED=0
560
- for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
561
- if mkdir "$LOCK" 2>/dev/null; then ACQUIRED=1; break; fi
562
- sleep 0.1
563
- done
564
- if [ "$ACQUIRED" -ne 1 ]; then echo 'ADMITTED=0'; echo 'REASON=admission lock busy'; exit 0; fi
565
- trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT
566
- if [ -e "$ROOT/.model-mode" ] || [ -e "$HOME/.model-mode" ]; then echo 'ADMITTED=0'; echo 'REASON=model mode is active'; exit 0; fi
567
- SLOT="$ROOT/.slots/${jobId}"
568
- if ! mkdir "$SLOT" 2>/dev/null; then echo 'ADMITTED=0'; echo 'REASON=job slot collision'; exit 0; fi
569
- printf '%s\n' "$NOW" > "$SLOT/created"
570
- ${requiresGpu ? 'touch "$SLOT/gpu"' : ":"}
571
- echo 'ADMITTED=1'
572
- `;
573
- }
574
-
575
- async function reserveWorker(worker: Worker, jobId: string, requiresGpu: boolean): Promise<{ admitted: boolean; reason?: string }> {
576
- const result = await runSsh(worker, "bash -s", { input: reserveScript(worker, jobId, requiresGpu), timeoutSeconds: 15 });
577
- if (result.code !== 0) return { admitted: false, reason: result.stderr.trim() || `slot reservation exited ${result.code}` };
578
- const admitted = /(^|\n)ADMITTED=1(\n|$)/.test(result.stdout);
579
- const reason = result.stdout.match(/(?:^|\n)REASON=([^\n]+)/)?.[1];
580
- return { admitted, reason };
581
- }
582
-
583
- async function releaseWorker(worker: Worker, jobId: string): Promise<void> {
584
- const script = `${rootAssignment(worker.root)}\nrm -rf "$ROOT/.slots/${jobId}"`;
585
- try {
586
- await runSsh(worker, "bash -s", { input: script, timeoutSeconds: 10 });
587
- } catch {
588
- // A stale slot is reaped automatically after six hours.
589
- }
590
- }
591
-
592
- function uploadCommand(worker: Worker, repoName: string, jobId: string): string {
593
- return `${rootAssignment(worker.root)}; JOB="$ROOT/${repoName}/${jobId}"; mkdir -p "$JOB/source"; date +%s > "$JOB/created"; tar -xzf - -C "$JOB/source"`;
594
- }
595
-
596
- function testScript(worker: Worker, repoName: string, jobId: string, command: string, keepSource: boolean): string {
597
- return `
598
- set -o pipefail
599
- ${rootAssignment(worker.root)}
600
- JOB="$ROOT/${repoName}/${jobId}"
601
- export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$HOME/.bun/bin:$HOME/.local/share/mise/shims:/opt/homebrew/opt/rustup/bin:/opt/homebrew/opt/openjdk@21/bin:/opt/homebrew/opt/ruby/bin:/opt/homebrew/lib/ruby/gems/4.0.0/bin:/opt/homebrew/opt/python@3.14/libexec/bin:/opt/homebrew/bin:/opt/homebrew/sbin:$PATH"
602
- if [ -d /opt/homebrew/opt/openjdk@21 ]; then export JAVA_HOME=/opt/homebrew/opt/openjdk@21; fi
603
- if [ -d /opt/homebrew/opt/dotnet/libexec ]; then export DOTNET_ROOT=/opt/homebrew/opt/dotnet/libexec; fi
604
- cd "$JOB/source" || exit 125
605
- START=$(date +%s)
606
- printf '{"state":"running","started":%s}\n' "$START" > "$JOB/status.json"
607
- set +e
608
- ( export CI=1; exec nice -n ${worker.nice} bash -c ${shQuote(command)} ) 2>&1 | tee "$JOB/test.log"
609
- CODE=\${PIPESTATUS[0]}
610
- END=$(date +%s)
611
- printf '{"state":"finished","exitCode":%s,"started":%s,"finished":%s,"durationSeconds":%s}\n' "$CODE" "$START" "$END" "$((END-START))" > "$JOB/result.json"
612
- cp "$JOB/result.json" "$JOB/status.json"
613
- ${keepSource ? "" : `
614
- # Drop the uploaded tree as soon as the run ends. Diagnostics (test.log,
615
- # result.json, status.json, created) are tiny and stay for the retention window.
616
- # Done here rather than client-side so it still happens if the SSH link drops.
617
- cd "$ROOT" 2>/dev/null || cd /
618
- rm -rf "$JOB/source"
619
- printf '1' > "$JOB/source-reclaimed"
620
- `}
621
- exit "$CODE"
622
- `;
623
- }
624
-
625
- function cleanupScript(worker: Worker, repoName: string): string {
626
- return `
627
- set +e
628
- ${rootAssignment(worker.root)}
629
- BASE="$ROOT/${repoName}"
630
- NOW=$(date +%s)
631
- MAX_AGE=${Math.floor(worker.retentionHours * 3600)}
632
- ORPHAN_AGE=${Math.floor(6 * 3600)}
633
- for d in "$BASE"/*; do
634
- [ -d "$d" ] || continue
635
- CREATED=$(cat "$d/created" 2>/dev/null || echo "$NOW")
636
- AGE=$((NOW-CREATED))
637
- # Whole job directory past its retention window.
638
- if [ "$AGE" -gt "$MAX_AGE" ]; then rm -rf "$d"; continue; fi
639
- # A source tree left by a job that was killed before it could self-clean.
640
- if [ -d "$d/source" ] && [ ! -f "$d/source-reclaimed" ] && [ "$AGE" -gt "$ORPHAN_AGE" ]; then
641
- rm -rf "$d/source"
642
- printf '1' > "$d/source-reclaimed"
643
- fi
644
- done
645
- `;
646
- }
647
-
648
- function blockedResult(worker: Worker | undefined, status: WorkerStatus | undefined, extra?: string) {
649
- const reason = extra || status?.reasons.join("; ") || "no eligible worker is ready";
650
- return {
651
- content: [
652
- {
653
- type: "text" as const,
654
- text: `REMOTE TEST BLOCKED${worker ? ` on ${worker.name}` : ""}: ${reason}. Run the test locally or choose another READY worker. Do not immediately retry the blocked worker.`,
655
- },
656
- ],
657
- details: { state: "blocked", worker: worker?.name, status, reason },
658
- };
659
- }
660
-
661
- const SnapshotSchema = Type.Object({
662
- mode: Type.Optional(
663
- StringEnum(["working-tree", "tracked", "paths"] as const, {
664
- description: '"working-tree" includes tracked and non-ignored untracked files; "tracked" includes tracked files only; "paths" includes only selected paths.',
665
- }),
666
- ),
667
- paths: Type.Optional(Type.Array(Type.String({ description: "Repository-relative file or directory" }), { maxItems: 500 })),
668
- excludePaths: Type.Optional(
669
- Type.Array(Type.String({ description: "Repository-relative file or directory to omit" }), { maxItems: 500 }),
670
- ),
671
- includeIgnored: Type.Optional(
672
- Type.Boolean({ description: "Include Git-ignored files. Default false; use only when an ignored fixture is explicitly required." }),
673
- ),
674
- });
675
-
676
- export default function remoteJobsExtension(pi: ExtensionAPI) {
677
- registerRemoteSetup(pi);
678
-
679
- pi.registerTool({
680
- name: "remote_status",
681
- label: "Remote Status",
682
- description: "Report live CPU, memory, GPU, disk, model-mode, and job-slot capacity for configured SSH workers.",
683
- promptSnippet: "Check available remote test/build worker capacity",
684
- parameters: Type.Object({
685
- host: Type.Optional(Type.String({ description: "Specific worker name; omit for all workers" })),
686
- }),
687
- async execute(_id, params) {
688
- const config = await loadConfig();
689
- const enabled = config.workers.filter((worker) => worker.enabled);
690
- if (enabled.length === 0) {
691
- return {
692
- content: [{ type: "text" as const, text: "No remote workers are enabled. Ask the user to run /remote setup to add or enable one." }],
693
- details: undefined,
694
- };
695
- }
696
- const selected = params.host ? enabled.filter((worker) => worker.name === params.host) : enabled;
697
- if (params.host && selected.length === 0) throw new Error(`Unknown remote worker: ${params.host}`);
698
- const statuses = await Promise.all(selected.map(probeWorker));
699
- statusCache = undefined;
700
- return {
701
- content: [{ type: "text", text: detailedStatuses(statuses) }],
702
- details: { statuses },
703
- };
704
- },
705
- });
706
-
707
- pi.registerTool({
708
- name: "remote_test",
709
- label: "Remote Test",
710
- description:
711
- "Snapshot the current local Git working tree (including uncommitted code), transfer the chosen files through SSH, and run a test/build command on a capacity-gated worker. The snapshot can be the full working tree, tracked files only, or explicit repository-relative paths.",
712
- promptSnippet: "Run tests/builds on a capacity-gated SSH worker using an exact local code snapshot",
713
- promptGuidelines: [
714
- "Call remote_test only after relevant edits have completed; never place remote_test before edit/write calls in the same tool batch.",
715
- "Use remote_test snapshot.mode=paths when only specific files and their manifests/lockfiles are required; otherwise use working-tree so current uncommitted code is tested.",
716
- "When remote_test reports BLOCKED, use another READY worker or run the command locally instead of immediately retrying that worker.",
717
- ],
718
- executionMode: "sequential",
719
- parameters: Type.Object({
720
- host: Type.Optional(Type.String({ description: 'Worker name or "auto". Default: auto.' })),
721
- command: Type.String({ minLength: 1, maxLength: 8192, description: "Test or build command to run at the snapshot root" }),
722
- repositoryPath: Type.Optional(
723
- Type.String({ description: "Local path inside the target Git repository. Default: current working directory." }),
724
- ),
725
- snapshot: Type.Optional(SnapshotSchema),
726
- requiresGpu: Type.Optional(Type.Boolean({ description: "Reserve an exclusive GPU job slot. Default false." })),
727
- timeoutSeconds: Type.Optional(
728
- Type.Integer({ minimum: 10, maximum: 21600, description: "Remote command timeout. Default 1800 seconds." }),
729
- ),
730
- keepSource: Type.Optional(
731
- Type.Boolean({
732
- description:
733
- "Keep the uploaded source tree on the worker after the run for debugging. "
734
- + "Default false: the tree is deleted on completion and only diagnostics are retained.",
735
- }),
736
- ),
737
- }),
738
- async execute(_id, params, signal, onUpdate, ctx) {
739
- const config = await loadConfig();
740
- const requestedHost = params.host ?? "auto";
741
- const enabledWorkers = config.workers.filter((worker) => worker.enabled);
742
- if (enabledWorkers.length === 0) {
743
- throw new Error("No remote workers are enabled. Ask the user to run /remote setup to add or enable one.");
744
- }
745
- if (requestedHost !== "auto" && !enabledWorkers.some((worker) => worker.name === requestedHost)) {
746
- throw new Error(`Unknown remote worker: ${requestedHost}. Available: ${enabledWorkers.map((w) => w.name).join(", ")}`);
747
- }
748
- let statuses = await probeWorkers(config, true);
749
- let worker = pickWorker(config, statuses, requestedHost, params.requiresGpu ?? false);
750
- if (!worker) return blockedResult(undefined, undefined, detailedStatuses(statuses));
751
- let workerStatus = statuses.find((status) => status.name === worker!.name);
752
- if (!workerStatus || workerStatus.state !== "ready") return blockedResult(worker, workerStatus);
753
-
754
- onUpdate?.({ content: [{ type: "text", text: `Creating selected snapshot for ${worker.name}...` }], details: { state: "snapshotting", worker: worker.name } });
755
- const snapshot = await createSnapshot(ctx.cwd, params);
756
- const jobId = `${new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14)}-${randomBytes(3).toString("hex")}`;
757
- let reserved = false;
758
- const started = Date.now();
759
- try {
760
- workerStatus = await probeWorker(worker);
761
- if (workerStatus.state !== "ready") return blockedResult(worker, workerStatus);
762
- const reservation = await reserveWorker(worker, jobId, params.requiresGpu ?? false);
763
- if (!reservation.admitted) return blockedResult(worker, workerStatus, reservation.reason);
764
- reserved = true;
765
- onUpdate?.({
766
- content: [
767
- {
768
- type: "text",
769
- text: `Uploading ${snapshot.files.length} files (${formatSize(snapshot.archiveBytes)}) to ${worker.name}...`,
770
- },
771
- ],
772
- details: { state: "uploading", worker: worker.name, jobId, snapshot: snapshot.fingerprint },
773
- });
774
- const upload = await runSsh(worker, uploadCommand(worker, snapshot.repoName, jobId), {
775
- input: { file: snapshot.archivePath },
776
- timeoutSeconds: Math.max(120, Math.min(900, Math.ceil(snapshot.archiveBytes / 250000) + 60)),
777
- signal,
778
- });
779
- if (upload.code !== 0 || upload.timedOut || upload.aborted) {
780
- throw new Error(upload.timedOut ? "snapshot upload timed out" : upload.stderr.trim() || `snapshot upload exited ${upload.code}`);
781
- }
782
- let streamed = "";
783
- let lastUpdate = 0;
784
- const run = await runSsh(worker, "bash -s", {
785
- input: testScript(worker, snapshot.repoName, jobId, params.command, params.keepSource === true),
786
- timeoutSeconds: params.timeoutSeconds ?? 1800,
787
- signal,
788
- onData: (chunk) => {
789
- streamed = appendTail(streamed, chunk);
790
- if (Date.now() - lastUpdate > 500) {
791
- lastUpdate = Date.now();
792
- const preview = truncateTail(streamed, { maxLines: 40, maxBytes: 8000 }).content;
793
- onUpdate?.({
794
- content: [{ type: "text", text: preview || `Running on ${worker!.name}...` }],
795
- details: { state: "running", worker: worker!.name, jobId },
796
- });
797
- }
798
- },
799
- });
800
- const durationSeconds = Math.round((Date.now() - started) / 100) / 10;
801
- const combined = [run.stdout, run.stderr].filter(Boolean).join("\n");
802
- const truncated = truncateTail(combined, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
803
- const remoteJob = `${worker.root}/${snapshot.repoName}/${jobId}`;
804
- const status = run.aborted ? "ABORTED" : run.timedOut ? "TIMED OUT" : run.code === 0 ? "PASSED" : "FAILED";
805
- let text = [
806
- `REMOTE TEST ${status}`,
807
- `Worker: ${worker.name}`,
808
- `Job: ${remoteJob}`,
809
- `Snapshot: ${snapshot.fingerprint.slice(0, 16)} (${snapshot.mode}, ${snapshot.files.length} files, ${formatSize(snapshot.archiveBytes)})`,
810
- `Git: ${snapshot.commit.slice(0, 12)}${snapshot.dirty ? " + working-tree changes" : ""}`,
811
- `Command: ${params.command}`,
812
- `Exit code: ${run.code}`,
813
- `Duration: ${durationSeconds}s`,
814
- "",
815
- truncated.content || "(no output)",
816
- ].join("\n");
817
- if (truncated.truncated || run.totalOutputBytes > DEFAULT_MAX_BYTES) {
818
- text += `\n\n[Output truncated; full log: ${remoteJob}/test.log]`;
819
- }
820
- void runSsh(worker, "bash -s", { input: cleanupScript(worker, snapshot.repoName), timeoutSeconds: 15 }).catch(() => {});
821
- statusCache = undefined;
822
- return {
823
- content: [{ type: "text", text }],
824
- details: {
825
- state: status.toLowerCase().replace(" ", "_"),
826
- worker: worker.name,
827
- jobId,
828
- remoteJob,
829
- exitCode: run.code,
830
- durationSeconds,
831
- timedOut: run.timedOut,
832
- aborted: run.aborted,
833
- snapshot: {
834
- mode: snapshot.mode,
835
- fingerprint: snapshot.fingerprint,
836
- commit: snapshot.commit,
837
- dirty: snapshot.dirty,
838
- fileCount: snapshot.files.length,
839
- files: snapshot.files,
840
- },
841
- },
842
- };
843
- } finally {
844
- await rm(snapshot.tempDir, { recursive: true, force: true });
845
- if (reserved) await releaseWorker(worker, jobId);
846
- }
847
- },
848
- });
849
-
850
- pi.on("before_agent_start", async (event) => {
851
- try {
852
- const config = await loadConfig();
853
- if (!config.injectStatus) return;
854
- const statuses = await probeWorkers(config);
855
- if (statuses.length === 0) return;
856
- const line = statuses.map(compactStatus).join("; ");
857
- return {
858
- systemPrompt:
859
- event.systemPrompt +
860
- `\n\nRemote worker capacity (recent sample): ${line}. remote_test always performs a fresh hard admission check. Never retry a BLOCKED worker immediately; choose another READY worker or run locally.`,
861
- };
862
- } catch {
863
- return;
864
- }
865
- });
866
- }
1
+ import { createHash, randomBytes } from "node:crypto";
2
+ import { createReadStream } from "node:fs";
3
+ import { lstat, mkdtemp, readlink, rm, writeFile } from "node:fs/promises";
4
+ import { basename, isAbsolute, relative, resolve, sep } from "node:path";
5
+ import { homedir, tmpdir } from "node:os";
6
+ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
7
+ import {
8
+ DEFAULT_MAX_BYTES,
9
+ DEFAULT_MAX_LINES,
10
+ formatSize,
11
+ truncateTail,
12
+ } from "@earendil-works/pi-coding-agent";
13
+ import { StringEnum } from "@earendil-works/pi-ai";
14
+ import { registerRemoteSetup } from "./setup.ts";
15
+ import { readRemote } from "./config-path.ts";
16
+ import { Type } from "typebox";
17
+ import {
18
+ appendTail,
19
+ shQuote,
20
+ rootAssignment,
21
+ runLocal,
22
+ runProcess,
23
+ runSshCommand,
24
+ type ProcessResult,
25
+ type RunOptions,
26
+ } from "../../core/exec/process.ts";
27
+
28
+ interface Limits {
29
+ cpuBlockPercent: number;
30
+ gpuBlockPercent: number;
31
+ gpuMemoryBlockPercent: number;
32
+ memoryBlockPercent: number;
33
+ minimumFreeDiskGB: number;
34
+ statusCacheSeconds: number;
35
+ retentionHours: number;
36
+ }
37
+
38
+ interface WorkerInput extends Partial<Limits> {
39
+ name: string;
40
+ ssh: string;
41
+ root?: string;
42
+ nice?: number;
43
+ tags?: string[];
44
+ /** Explicit key path, for hosts added without an ~/.ssh/config entry. */
45
+ identityFile?: string;
46
+ port?: number;
47
+ /** Toggled off in the picker without losing the entry. */
48
+ enabled?: boolean;
49
+ }
50
+
51
+ // WorkerInput carries the same limit keys as optional overrides, so they must be
52
+ // stripped before re-declaring them as required or the two bases conflict.
53
+ interface Worker extends Omit<WorkerInput, keyof Limits | "enabled">, Limits {
54
+ enabled: boolean;
55
+ root: string;
56
+ nice: number;
57
+ tags: string[];
58
+ }
59
+
60
+ interface Config {
61
+ injectStatus: boolean;
62
+ workers: Worker[];
63
+ }
64
+
65
+ interface WorkerStatus {
66
+ name: string;
67
+ ssh: string;
68
+ os?: string;
69
+ state: "ready" | "blocked" | "unreachable";
70
+ cpuPercent?: number;
71
+ memoryPercent?: number;
72
+ gpuPercent?: number;
73
+ gpuMemoryPercent?: number;
74
+ freeDiskGB?: number;
75
+ jobs?: number;
76
+ gpuJobs?: number;
77
+ thermal?: string;
78
+ modelMode?: boolean;
79
+ reasons: string[];
80
+ tags: string[];
81
+ sampledAt: string;
82
+ }
83
+
84
+ interface SnapshotResult {
85
+ tempDir: string;
86
+ archivePath: string;
87
+ repoRoot: string;
88
+ repoName: string;
89
+ mode: "working-tree" | "tracked" | "paths";
90
+ files: string[];
91
+ fingerprint: string;
92
+ archiveBytes: number;
93
+ commit: string;
94
+ dirty: boolean;
95
+ }
96
+
97
+ const DEFAULT_LIMITS: Limits = {
98
+ cpuBlockPercent: 90,
99
+ gpuBlockPercent: 90,
100
+ gpuMemoryBlockPercent: 90,
101
+ memoryBlockPercent: 90,
102
+ minimumFreeDiskGB: 10,
103
+ statusCacheSeconds: 30,
104
+ retentionHours: 24,
105
+ };
106
+
107
+ const HARD_WALK_EXCLUDES = new Set([
108
+ ".git",
109
+ "node_modules",
110
+ ".venv",
111
+ "__pycache__",
112
+ "target",
113
+ "dist",
114
+ "build",
115
+ "coverage",
116
+ ".next",
117
+ "remote_tests",
118
+ ]);
119
+
120
+ let statusCache: { expiresAt: number; key: string; statuses: WorkerStatus[] } | undefined;
121
+
122
+ function finiteNumber(value: unknown, fallback: number, min: number, max: number): number {
123
+ return typeof value === "number" && Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback;
124
+ }
125
+
126
+ async function loadConfig(): Promise<Config> {
127
+ // The `remote` section of pi-plus.json. An empty worker list is the normal
128
+ // first-run state: /remote setup populates it.
129
+ const parsed = readRemote();
130
+ const defaults = parsed.defaults ?? {};
131
+ const defaultLimits: Limits = {
132
+ cpuBlockPercent: finiteNumber(defaults.cpuBlockPercent, DEFAULT_LIMITS.cpuBlockPercent, 1, 100),
133
+ gpuBlockPercent: finiteNumber(defaults.gpuBlockPercent, DEFAULT_LIMITS.gpuBlockPercent, 1, 100),
134
+ gpuMemoryBlockPercent: finiteNumber(
135
+ defaults.gpuMemoryBlockPercent,
136
+ DEFAULT_LIMITS.gpuMemoryBlockPercent,
137
+ 1,
138
+ 100,
139
+ ),
140
+ memoryBlockPercent: finiteNumber(defaults.memoryBlockPercent, DEFAULT_LIMITS.memoryBlockPercent, 1, 100),
141
+ minimumFreeDiskGB: finiteNumber(defaults.minimumFreeDiskGB, DEFAULT_LIMITS.minimumFreeDiskGB, 0, 100000),
142
+ statusCacheSeconds: finiteNumber(defaults.statusCacheSeconds, DEFAULT_LIMITS.statusCacheSeconds, 1, 600),
143
+ retentionHours: finiteNumber(defaults.retentionHours, DEFAULT_LIMITS.retentionHours, 1, 24 * 365),
144
+ };
145
+ const names = new Set<string>();
146
+ const workers: Worker[] = parsed.workers.map((raw: WorkerInput, index: number) => {
147
+ if (!raw || typeof raw.name !== "string" || !raw.name.trim() || typeof raw.ssh !== "string" || !raw.ssh.trim()) {
148
+ throw new Error(`Invalid worker at index ${index}: name and ssh are required strings.`);
149
+ }
150
+ if (names.has(raw.name)) throw new Error(`Duplicate remote worker name: ${raw.name}`);
151
+ names.add(raw.name);
152
+ const root = typeof raw.root === "string" && raw.root.trim() ? raw.root.trim() : "~/remote_tests";
153
+ if (root.includes("\n") || root.includes("\0") || (!root.startsWith("~/") && !root.startsWith("/"))) {
154
+ throw new Error(`Worker ${raw.name} root must be an absolute POSIX path or start with ~/`);
155
+ }
156
+ return {
157
+ name: raw.name.trim(),
158
+ ssh: raw.ssh.trim(),
159
+ root,
160
+ enabled: raw.enabled !== false,
161
+ identityFile: typeof raw.identityFile === "string" && raw.identityFile.trim() ? raw.identityFile.trim() : undefined,
162
+ port: Number.isInteger(raw.port) && raw.port! > 0 && raw.port! < 65536 ? raw.port : undefined,
163
+ nice: Math.floor(finiteNumber(raw.nice, 10, 0, 19)),
164
+ tags: Array.isArray(raw.tags) ? raw.tags.filter((tag): tag is string => typeof tag === "string") : [],
165
+ cpuBlockPercent: finiteNumber(raw.cpuBlockPercent, defaultLimits.cpuBlockPercent, 1, 100),
166
+ gpuBlockPercent: finiteNumber(raw.gpuBlockPercent, defaultLimits.gpuBlockPercent, 1, 100),
167
+ gpuMemoryBlockPercent: finiteNumber(
168
+ raw.gpuMemoryBlockPercent,
169
+ defaultLimits.gpuMemoryBlockPercent,
170
+ 1,
171
+ 100,
172
+ ),
173
+ memoryBlockPercent: finiteNumber(raw.memoryBlockPercent, defaultLimits.memoryBlockPercent, 1, 100),
174
+ minimumFreeDiskGB: finiteNumber(raw.minimumFreeDiskGB, defaultLimits.minimumFreeDiskGB, 0, 100000),
175
+ statusCacheSeconds: finiteNumber(raw.statusCacheSeconds, defaultLimits.statusCacheSeconds, 1, 600),
176
+ retentionHours: finiteNumber(raw.retentionHours, defaultLimits.retentionHours, 1, 24 * 365),
177
+ };
178
+ });
179
+ return { injectStatus: parsed.injectStatus !== false, workers };
180
+ }
181
+
182
+ /** `-i`/`-p` only when the worker was added without an ~/.ssh/config entry. */
183
+ function sshArgsFor(worker: Worker): string[] {
184
+ const args: string[] = [];
185
+ if (worker.identityFile) {
186
+ const expanded = worker.identityFile.startsWith("~/")
187
+ ? resolve(homedir(), worker.identityFile.slice(2))
188
+ : worker.identityFile;
189
+ args.push("-i", expanded, "-o", "IdentitiesOnly=yes");
190
+ }
191
+ if (worker.port) args.push("-p", String(worker.port));
192
+ return args;
193
+ }
194
+
195
+ async function runSsh(
196
+ worker: Worker,
197
+ remoteCommand: string,
198
+ options: RunOptions = {},
199
+ ): Promise<ProcessResult> {
200
+ return runSshCommand(worker.ssh, remoteCommand, options, sshArgsFor(worker));
201
+ }
202
+
203
+ function healthScript(worker: Worker): string {
204
+ return `
205
+ set +e
206
+ LC_ALL=C
207
+ ${rootAssignment(worker.root)}
208
+ OS=$(uname -s 2>/dev/null || echo unknown)
209
+ CPU=-1
210
+ MEM=-1
211
+ GPU=-1
212
+ GPU_MEM=-1
213
+ THERMAL=normal
214
+ if [ "$OS" = Darwin ]; then
215
+ CORES=$(sysctl -n hw.logicalcpu 2>/dev/null || echo 1)
216
+ CPU=$(ps -A -o %cpu= 2>/dev/null | awk -v c="$CORES" '{s+=$1} END {if(c<1)c=1; v=s/c; if(v>100)v=100; printf "%.1f",v}')
217
+ FREE=$(memory_pressure -Q 2>/dev/null | awk -F': ' '/System-wide memory free percentage/ {gsub(/%/,"",$2); print $2; exit}')
218
+ if [ -n "$FREE" ]; then MEM=$(awk -v f="$FREE" 'BEGIN {printf "%.1f",100-f}'); fi
219
+ if ! pmset -g therm 2>/dev/null | grep -q 'No thermal warning level'; then THERMAL=warning; fi
220
+ else
221
+ set -- $(awk '/^cpu / {idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i; print total,idle; exit}' /proc/stat 2>/dev/null)
222
+ T1=$1; I1=$2
223
+ sleep 0.4
224
+ set -- $(awk '/^cpu / {idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i; print total,idle; exit}' /proc/stat 2>/dev/null)
225
+ T2=$1; I2=$2
226
+ if [ -n "$T1" ] && [ "$T2" -gt "$T1" ] 2>/dev/null; then CPU=$(awk -v t1="$T1" -v i1="$I1" -v t2="$T2" -v i2="$I2" 'BEGIN {printf "%.1f",100*(1-(i2-i1)/(t2-t1))}'); fi
227
+ MEM=$(awk '/MemTotal/ {t=$2} /MemAvailable/ {a=$2} END {if(t>0) printf "%.1f",100*(t-a)/t; else print -1}' /proc/meminfo 2>/dev/null)
228
+ if command -v nvidia-smi >/dev/null 2>&1; then
229
+ GPU_LINE=$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')
230
+ GPU=$(printf '%s' "$GPU_LINE" | cut -d, -f1)
231
+ GPU_USED=$(printf '%s' "$GPU_LINE" | cut -d, -f2)
232
+ GPU_TOTAL=$(printf '%s' "$GPU_LINE" | cut -d, -f3)
233
+ if [ -n "$GPU_TOTAL" ] && [ "$GPU_TOTAL" -gt 0 ] 2>/dev/null; then GPU_MEM=$(awk -v u="$GPU_USED" -v t="$GPU_TOTAL" 'BEGIN {printf "%.1f",100*u/t}'); fi
234
+ fi
235
+ fi
236
+ FREE_KB=$(df -Pk "$HOME" 2>/dev/null | awk 'NR==2 {print $4}')
237
+ FREE_GB=$(awk -v k="\${FREE_KB:-0}" 'BEGIN {printf "%.1f",k/1048576}')
238
+ JOBS=0
239
+ GPU_JOBS=0
240
+ for d in "$ROOT/.slots"/*; do
241
+ [ -d "$d" ] || continue
242
+ JOBS=$((JOBS+1))
243
+ [ -f "$d/gpu" ] && GPU_JOBS=$((GPU_JOBS+1))
244
+ done
245
+ MODEL_MODE=0
246
+ if [ -e "$ROOT/.model-mode" ] || [ -e "$HOME/.model-mode" ]; then MODEL_MODE=1; fi
247
+ printf 'OS=%s\nCPU=%s\nMEM=%s\nGPU=%s\nGPU_MEM=%s\nFREE_GB=%s\nJOBS=%s\nGPU_JOBS=%s\nTHERMAL=%s\nMODEL_MODE=%s\n' "$OS" "$CPU" "$MEM" "$GPU" "$GPU_MEM" "$FREE_GB" "$JOBS" "$GPU_JOBS" "$THERMAL" "$MODEL_MODE"
248
+ `;
249
+ }
250
+
251
+ function parseNumber(value: string | undefined): number | undefined {
252
+ if (value === undefined) return undefined;
253
+ const parsed = Number(value);
254
+ return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
255
+ }
256
+
257
+ async function probeWorker(worker: Worker): Promise<WorkerStatus> {
258
+ const sampledAt = new Date().toISOString();
259
+ try {
260
+ const result = await runSsh(worker, "bash -s", { input: healthScript(worker), timeoutSeconds: 12 });
261
+ if (result.code !== 0 || result.timedOut || result.aborted) {
262
+ const message = result.timedOut ? "health check timed out" : (result.stderr.trim() || `SSH exited ${result.code}`);
263
+ return {
264
+ name: worker.name,
265
+ ssh: worker.ssh,
266
+ state: "unreachable",
267
+ reasons: [message],
268
+ tags: worker.tags,
269
+ sampledAt,
270
+ };
271
+ }
272
+ const values = new Map<string, string>();
273
+ for (const line of result.stdout.split(/\r?\n/)) {
274
+ const separator = line.indexOf("=");
275
+ if (separator > 0) values.set(line.slice(0, separator), line.slice(separator + 1));
276
+ }
277
+ const cpuPercent = parseNumber(values.get("CPU"));
278
+ const memoryPercent = parseNumber(values.get("MEM"));
279
+ const gpuPercent = parseNumber(values.get("GPU"));
280
+ const gpuMemoryPercent = parseNumber(values.get("GPU_MEM"));
281
+ const freeDiskGB = parseNumber(values.get("FREE_GB"));
282
+ const jobs = parseNumber(values.get("JOBS")) ?? 0;
283
+ const gpuJobs = parseNumber(values.get("GPU_JOBS")) ?? 0;
284
+ const thermal = values.get("THERMAL") || "unknown";
285
+ const modelMode = values.get("MODEL_MODE") === "1";
286
+ const reasons: string[] = [];
287
+ if (modelMode) reasons.push("model mode is active");
288
+ if (cpuPercent !== undefined && cpuPercent >= worker.cpuBlockPercent) {
289
+ reasons.push(`CPU ${cpuPercent.toFixed(1)}% >= ${worker.cpuBlockPercent}%`);
290
+ }
291
+ if (memoryPercent !== undefined && memoryPercent >= worker.memoryBlockPercent) {
292
+ reasons.push(`memory ${memoryPercent.toFixed(1)}% >= ${worker.memoryBlockPercent}%`);
293
+ }
294
+ if (gpuPercent !== undefined && gpuPercent >= worker.gpuBlockPercent) {
295
+ reasons.push(`GPU ${gpuPercent.toFixed(1)}% >= ${worker.gpuBlockPercent}%`);
296
+ }
297
+ if (gpuMemoryPercent !== undefined && gpuMemoryPercent >= worker.gpuMemoryBlockPercent) {
298
+ reasons.push(`GPU memory ${gpuMemoryPercent.toFixed(1)}% >= ${worker.gpuMemoryBlockPercent}%`);
299
+ }
300
+ if (freeDiskGB !== undefined && freeDiskGB < worker.minimumFreeDiskGB) {
301
+ reasons.push(`free disk ${freeDiskGB.toFixed(1)} GB < ${worker.minimumFreeDiskGB} GB`);
302
+ }
303
+ if (thermal !== "normal") reasons.push(`thermal state ${thermal}`);
304
+ return {
305
+ name: worker.name,
306
+ ssh: worker.ssh,
307
+ os: values.get("OS"),
308
+ state: reasons.length ? "blocked" : "ready",
309
+ cpuPercent,
310
+ memoryPercent,
311
+ gpuPercent,
312
+ gpuMemoryPercent,
313
+ freeDiskGB,
314
+ jobs,
315
+ gpuJobs,
316
+ thermal,
317
+ modelMode,
318
+ reasons,
319
+ tags: worker.tags,
320
+ sampledAt,
321
+ };
322
+ } catch (error) {
323
+ return {
324
+ name: worker.name,
325
+ ssh: worker.ssh,
326
+ state: "unreachable",
327
+ reasons: [error instanceof Error ? error.message : String(error)],
328
+ tags: worker.tags,
329
+ sampledAt,
330
+ };
331
+ }
332
+ }
333
+
334
+ async function probeWorkers(config: Config, force = false): Promise<WorkerStatus[]> {
335
+ const active = config.workers.filter((worker) => worker.enabled);
336
+ if (active.length === 0) return [];
337
+ const key = JSON.stringify(active.map((worker) => [worker.name, worker.ssh, worker.root]));
338
+ const ttlSeconds = Math.min(...active.map((worker) => worker.statusCacheSeconds));
339
+ if (!force && statusCache && statusCache.key === key && statusCache.expiresAt > Date.now()) return statusCache.statuses;
340
+ const statuses = await Promise.all(active.map(probeWorker));
341
+ statusCache = { key, statuses, expiresAt: Date.now() + ttlSeconds * 1000 };
342
+ return statuses;
343
+ }
344
+
345
+ function metric(value: number | undefined): string {
346
+ return value === undefined ? "?" : `${Math.round(value)}%`;
347
+ }
348
+
349
+ function compactStatus(status: WorkerStatus): string {
350
+ const gpu = status.gpuPercent === undefined ? "GPU ?" : `GPU ${metric(status.gpuPercent)}`;
351
+ const reason = status.reasons.length ? ` (${status.reasons.join(", ")})` : "";
352
+ return `${status.name} ${status.state.toUpperCase()} CPU ${metric(status.cpuPercent)} MEM ${metric(status.memoryPercent)} ${gpu} active-jobs ${status.jobs ?? "?"}${reason}`;
353
+ }
354
+
355
+ function detailedStatuses(statuses: WorkerStatus[]): string {
356
+ return statuses
357
+ .map((status) => {
358
+ const fields = [
359
+ `${status.name.padEnd(10)} ${status.state.toUpperCase().padEnd(11)}`,
360
+ `CPU ${metric(status.cpuPercent).padStart(4)}`,
361
+ `MEM ${metric(status.memoryPercent).padStart(4)}`,
362
+ `GPU ${metric(status.gpuPercent).padStart(4)}`,
363
+ `VRAM ${metric(status.gpuMemoryPercent).padStart(4)}`,
364
+ `disk ${status.freeDiskGB === undefined ? "?" : `${status.freeDiskGB.toFixed(1)}GB`}`,
365
+ `active-jobs ${status.jobs ?? "?"}`,
366
+ ];
367
+ return fields.join(" ") + (status.reasons.length ? `\n blocked: ${status.reasons.join("; ")}` : "");
368
+ })
369
+ .join("\n");
370
+ }
371
+
372
+ function normalizeRepoPath(input: string): string {
373
+ const value = input.replace(/\\/g, "/").replace(/^\.\//, "").replace(/\/+$/, "");
374
+ if (value === "." || value === "") return "";
375
+ if (value.includes("\0") || value.includes("\n") || isAbsolute(value) || /^[A-Za-z]:/.test(value)) {
376
+ throw new Error(`Snapshot path must be repository-relative: ${input}`);
377
+ }
378
+ const parts = value.split("/");
379
+ if (parts.some((part) => part === "..")) throw new Error(`Snapshot path escapes the repository: ${input}`);
380
+ if (parts.some((part) => part === ".git")) throw new Error(`Snapshot paths cannot include .git: ${input}`);
381
+ return parts.filter((part) => part && part !== ".").join("/");
382
+ }
383
+
384
+ function matchesPath(file: string, selected: string): boolean {
385
+ return selected === "" || file === selected || file.startsWith(`${selected}/`);
386
+ }
387
+
388
+ async function walkFiles(root: string, current = ""): Promise<string[]> {
389
+ const directory = resolve(root, current || ".");
390
+ const entries = await import("node:fs/promises").then(({ readdir }) => readdir(directory, { withFileTypes: true }));
391
+ const files: string[] = [];
392
+ for (const entry of entries) {
393
+ if (HARD_WALK_EXCLUDES.has(entry.name)) continue;
394
+ const rel = current ? `${current}/${entry.name}` : entry.name;
395
+ if (entry.isDirectory()) files.push(...(await walkFiles(root, rel)));
396
+ else if (entry.isFile() || entry.isSymbolicLink()) files.push(rel);
397
+ }
398
+ return files;
399
+ }
400
+
401
+ async function gitOutput(repoRoot: string, args: string[]): Promise<string> {
402
+ const result = await runLocal("git", ["-C", repoRoot, ...args]);
403
+ if (result.code !== 0) throw new Error(result.stderr.trim() || `git ${args.join(" ")} failed`);
404
+ return result.stdout;
405
+ }
406
+
407
+ async function resolveRepository(repositoryPath: string | undefined, cwd: string): Promise<string> {
408
+ const candidate = resolve(cwd, repositoryPath?.replace(/^@/, "") || ".");
409
+ const result = await runLocal("git", ["-C", candidate, "rev-parse", "--show-toplevel"]);
410
+ if (result.code !== 0) throw new Error(`${candidate} is not inside a Git repository.`);
411
+ return resolve(result.stdout.trim());
412
+ }
413
+
414
+ async function selectFiles(
415
+ repoRoot: string,
416
+ mode: "working-tree" | "tracked" | "paths",
417
+ paths: string[],
418
+ excludes: string[],
419
+ includeIgnored: boolean,
420
+ ): Promise<string[]> {
421
+ let files: string[];
422
+ if (includeIgnored && mode !== "tracked") {
423
+ files = await walkFiles(repoRoot);
424
+ } else {
425
+ const args = mode === "tracked" ? ["ls-files", "-z", "--cached"] : ["ls-files", "-z", "--cached", "--others", "--exclude-standard"];
426
+ files = (await gitOutput(repoRoot, args)).split("\0").filter(Boolean).map((file) => file.replace(/\\/g, "/"));
427
+ }
428
+ const normalizedPaths = paths.map(normalizeRepoPath);
429
+ const normalizedExcludes = excludes.map(normalizeRepoPath);
430
+ if (mode === "paths") {
431
+ if (normalizedPaths.length === 0) throw new Error('snapshot.mode "paths" requires at least one path.');
432
+ files = files.filter((file) => normalizedPaths.some((selected) => matchesPath(file, selected)));
433
+ }
434
+ files = files.filter((file) => !normalizedExcludes.some((excluded) => matchesPath(file, excluded)));
435
+ const existing: string[] = [];
436
+ for (const file of [...new Set(files)].sort()) {
437
+ const absolute = resolve(repoRoot, file.split("/").join(sep));
438
+ if (relative(repoRoot, absolute).startsWith("..")) continue;
439
+ try {
440
+ const stat = await lstat(absolute);
441
+ if (stat.isFile() || stat.isSymbolicLink()) existing.push(file);
442
+ } catch {
443
+ // Deleted tracked files are intentionally absent from the snapshot.
444
+ }
445
+ }
446
+ if (existing.length === 0) throw new Error("Snapshot selection contains no files.");
447
+ return existing;
448
+ }
449
+
450
+ async function fileStamp(repoRoot: string, files: string[]): Promise<string> {
451
+ const hash = createHash("sha256");
452
+ for (const file of files) {
453
+ const absolute = resolve(repoRoot, file.split("/").join(sep));
454
+ const stat = await lstat(absolute);
455
+ hash.update(file).update("\0").update(`${stat.size}:${stat.mtimeMs}:${stat.mode}`).update("\0");
456
+ if (stat.isSymbolicLink()) hash.update(await readlink(absolute));
457
+ }
458
+ return hash.digest("hex");
459
+ }
460
+
461
+ async function sha256File(path: string): Promise<string> {
462
+ return new Promise((resolvePromise, reject) => {
463
+ const hash = createHash("sha256");
464
+ const stream = createReadStream(path);
465
+ stream.on("data", (chunk) => hash.update(chunk));
466
+ stream.on("error", reject);
467
+ stream.on("end", () => resolvePromise(hash.digest("hex")));
468
+ });
469
+ }
470
+
471
+ async function createSnapshot(
472
+ cwd: string,
473
+ params: {
474
+ repositoryPath?: string;
475
+ snapshot?: {
476
+ mode?: "working-tree" | "tracked" | "paths";
477
+ paths?: string[];
478
+ excludePaths?: string[];
479
+ includeIgnored?: boolean;
480
+ };
481
+ },
482
+ ): Promise<SnapshotResult> {
483
+ const repoRoot = await resolveRepository(params.repositoryPath, cwd);
484
+ const mode = params.snapshot?.mode ?? "working-tree";
485
+ const paths = params.snapshot?.paths ?? [];
486
+ const excludes = params.snapshot?.excludePaths ?? [];
487
+ const includeIgnored = params.snapshot?.includeIgnored ?? false;
488
+ const repoName = basename(repoRoot).replace(/[^A-Za-z0-9._-]+/g, "-") || "repo";
489
+ const tempDir = await mkdtemp(resolve(tmpdir(), "pi-remote-test-"));
490
+ const archivePath = resolve(tempDir, "snapshot.tar.gz");
491
+ try {
492
+ for (let attempt = 1; attempt <= 2; attempt++) {
493
+ const files = await selectFiles(repoRoot, mode, paths, excludes, includeIgnored);
494
+ const before = await fileStamp(repoRoot, files);
495
+ const listPath = resolve(tempDir, "files.nul");
496
+ await writeFile(listPath, Buffer.from(`${files.join("\0")}\0`, "utf8"), { mode: 0o600 });
497
+ const tar = await runProcess("tar", ["-czf", archivePath, "-C", repoRoot, "--no-recursion", "--null", "-T", listPath], {
498
+ timeoutSeconds: 300,
499
+ });
500
+ if (tar.code !== 0) throw new Error(tar.stderr.trim() || "Failed to create snapshot archive.");
501
+ const after = await fileStamp(repoRoot, files);
502
+ if (before !== after) {
503
+ if (attempt === 2) throw new Error("Selected files changed while the snapshot was being created; retry after edits settle.");
504
+ continue;
505
+ }
506
+ const stat = await lstat(archivePath);
507
+ const commit = (await gitOutput(repoRoot, ["rev-parse", "HEAD"])).trim();
508
+ const dirty = (await gitOutput(repoRoot, ["status", "--porcelain", "--untracked-files=normal"])).length > 0;
509
+ return {
510
+ tempDir,
511
+ archivePath,
512
+ repoRoot,
513
+ repoName,
514
+ mode,
515
+ files,
516
+ fingerprint: await sha256File(archivePath),
517
+ archiveBytes: stat.size,
518
+ commit,
519
+ dirty,
520
+ };
521
+ }
522
+ throw new Error("Snapshot creation failed.");
523
+ } catch (error) {
524
+ await rm(tempDir, { recursive: true, force: true });
525
+ throw error;
526
+ }
527
+ }
528
+
529
+ function gpuCapable(worker: Worker): boolean {
530
+ return worker.tags.some((tag) => ["gpu", "cuda", "metal"].includes(tag.toLowerCase()));
531
+ }
532
+
533
+ function pickWorker(config: Config, statuses: WorkerStatus[], requested: string, requiresGpu: boolean): Worker | undefined {
534
+ if (requested !== "auto") return config.workers.find((worker) => worker.enabled && worker.name === requested);
535
+ const ready = statuses
536
+ .filter((status) => status.state === "ready")
537
+ .map((status) => ({ status, worker: config.workers.find((worker) => worker.name === status.name)! }))
538
+ .filter(({ worker }) => worker && (!requiresGpu || gpuCapable(worker)));
539
+ ready.sort((a, b) => {
540
+ const score = (item: (typeof ready)[number]) =>
541
+ Math.max(item.status.cpuPercent ?? 0, item.status.memoryPercent ?? 0, item.status.gpuPercent ?? 0);
542
+ return score(a) - score(b);
543
+ });
544
+ return ready[0]?.worker;
545
+ }
546
+
547
+ function reserveScript(worker: Worker, jobId: string, requiresGpu: boolean): string {
548
+ return `
549
+ set -u
550
+ ${rootAssignment(worker.root)}
551
+ mkdir -p "$ROOT/.slots"
552
+ LOCK="$ROOT/.admission.lock"
553
+ NOW=$(date +%s)
554
+ for d in "$ROOT/.slots"/*; do
555
+ [ -d "$d" ] || continue
556
+ CREATED=$(cat "$d/created" 2>/dev/null || echo "$NOW")
557
+ if [ $((NOW-CREATED)) -gt 21600 ]; then rm -rf "$d"; fi
558
+ done
559
+ ACQUIRED=0
560
+ for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
561
+ if mkdir "$LOCK" 2>/dev/null; then ACQUIRED=1; break; fi
562
+ sleep 0.1
563
+ done
564
+ if [ "$ACQUIRED" -ne 1 ]; then echo 'ADMITTED=0'; echo 'REASON=admission lock busy'; exit 0; fi
565
+ trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT
566
+ if [ -e "$ROOT/.model-mode" ] || [ -e "$HOME/.model-mode" ]; then echo 'ADMITTED=0'; echo 'REASON=model mode is active'; exit 0; fi
567
+ SLOT="$ROOT/.slots/${jobId}"
568
+ if ! mkdir "$SLOT" 2>/dev/null; then echo 'ADMITTED=0'; echo 'REASON=job slot collision'; exit 0; fi
569
+ printf '%s\n' "$NOW" > "$SLOT/created"
570
+ ${requiresGpu ? 'touch "$SLOT/gpu"' : ":"}
571
+ echo 'ADMITTED=1'
572
+ `;
573
+ }
574
+
575
+ async function reserveWorker(worker: Worker, jobId: string, requiresGpu: boolean): Promise<{ admitted: boolean; reason?: string }> {
576
+ const result = await runSsh(worker, "bash -s", { input: reserveScript(worker, jobId, requiresGpu), timeoutSeconds: 15 });
577
+ if (result.code !== 0) return { admitted: false, reason: result.stderr.trim() || `slot reservation exited ${result.code}` };
578
+ const admitted = /(^|\n)ADMITTED=1(\n|$)/.test(result.stdout);
579
+ const reason = result.stdout.match(/(?:^|\n)REASON=([^\n]+)/)?.[1];
580
+ return { admitted, reason };
581
+ }
582
+
583
+ async function releaseWorker(worker: Worker, jobId: string): Promise<void> {
584
+ const script = `${rootAssignment(worker.root)}\nrm -rf "$ROOT/.slots/${jobId}"`;
585
+ try {
586
+ await runSsh(worker, "bash -s", { input: script, timeoutSeconds: 10 });
587
+ } catch {
588
+ // A stale slot is reaped automatically after six hours.
589
+ }
590
+ }
591
+
592
+ function uploadCommand(worker: Worker, repoName: string, jobId: string): string {
593
+ return `${rootAssignment(worker.root)}; JOB="$ROOT/${repoName}/${jobId}"; mkdir -p "$JOB/source"; date +%s > "$JOB/created"; tar -xzf - -C "$JOB/source"`;
594
+ }
595
+
596
+ function testScript(worker: Worker, repoName: string, jobId: string, command: string, keepSource: boolean): string {
597
+ return `
598
+ set -o pipefail
599
+ ${rootAssignment(worker.root)}
600
+ JOB="$ROOT/${repoName}/${jobId}"
601
+ export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$HOME/.bun/bin:$HOME/.local/share/mise/shims:/opt/homebrew/opt/rustup/bin:/opt/homebrew/opt/openjdk@21/bin:/opt/homebrew/opt/ruby/bin:/opt/homebrew/lib/ruby/gems/4.0.0/bin:/opt/homebrew/opt/python@3.14/libexec/bin:/opt/homebrew/bin:/opt/homebrew/sbin:$PATH"
602
+ if [ -d /opt/homebrew/opt/openjdk@21 ]; then export JAVA_HOME=/opt/homebrew/opt/openjdk@21; fi
603
+ if [ -d /opt/homebrew/opt/dotnet/libexec ]; then export DOTNET_ROOT=/opt/homebrew/opt/dotnet/libexec; fi
604
+ cd "$JOB/source" || exit 125
605
+ START=$(date +%s)
606
+ printf '{"state":"running","started":%s}\n' "$START" > "$JOB/status.json"
607
+ set +e
608
+ ( export CI=1; exec nice -n ${worker.nice} bash -c ${shQuote(command)} ) 2>&1 | tee "$JOB/test.log"
609
+ CODE=\${PIPESTATUS[0]}
610
+ END=$(date +%s)
611
+ printf '{"state":"finished","exitCode":%s,"started":%s,"finished":%s,"durationSeconds":%s}\n' "$CODE" "$START" "$END" "$((END-START))" > "$JOB/result.json"
612
+ cp "$JOB/result.json" "$JOB/status.json"
613
+ ${keepSource ? "" : `
614
+ # Drop the uploaded tree as soon as the run ends. Diagnostics (test.log,
615
+ # result.json, status.json, created) are tiny and stay for the retention window.
616
+ # Done here rather than client-side so it still happens if the SSH link drops.
617
+ cd "$ROOT" 2>/dev/null || cd /
618
+ rm -rf "$JOB/source"
619
+ printf '1' > "$JOB/source-reclaimed"
620
+ `}
621
+ exit "$CODE"
622
+ `;
623
+ }
624
+
625
+ function cleanupScript(worker: Worker, repoName: string): string {
626
+ return `
627
+ set +e
628
+ ${rootAssignment(worker.root)}
629
+ BASE="$ROOT/${repoName}"
630
+ NOW=$(date +%s)
631
+ MAX_AGE=${Math.floor(worker.retentionHours * 3600)}
632
+ ORPHAN_AGE=${Math.floor(6 * 3600)}
633
+ for d in "$BASE"/*; do
634
+ [ -d "$d" ] || continue
635
+ CREATED=$(cat "$d/created" 2>/dev/null || echo "$NOW")
636
+ AGE=$((NOW-CREATED))
637
+ # Whole job directory past its retention window.
638
+ if [ "$AGE" -gt "$MAX_AGE" ]; then rm -rf "$d"; continue; fi
639
+ # A source tree left by a job that was killed before it could self-clean.
640
+ if [ -d "$d/source" ] && [ ! -f "$d/source-reclaimed" ] && [ "$AGE" -gt "$ORPHAN_AGE" ]; then
641
+ rm -rf "$d/source"
642
+ printf '1' > "$d/source-reclaimed"
643
+ fi
644
+ done
645
+ `;
646
+ }
647
+
648
+ function blockedResult(worker: Worker | undefined, status: WorkerStatus | undefined, extra?: string) {
649
+ const reason = extra || status?.reasons.join("; ") || "no eligible worker is ready";
650
+ return {
651
+ content: [
652
+ {
653
+ type: "text" as const,
654
+ text: `REMOTE TEST BLOCKED${worker ? ` on ${worker.name}` : ""}: ${reason}. Run the test locally or choose another READY worker. Do not immediately retry the blocked worker.`,
655
+ },
656
+ ],
657
+ details: { state: "blocked", worker: worker?.name, status, reason },
658
+ };
659
+ }
660
+
661
+ const SnapshotSchema = Type.Object({
662
+ mode: Type.Optional(
663
+ StringEnum(["working-tree", "tracked", "paths"] as const, {
664
+ description: '"working-tree" includes tracked and non-ignored untracked files; "tracked" includes tracked files only; "paths" includes only selected paths.',
665
+ }),
666
+ ),
667
+ paths: Type.Optional(Type.Array(Type.String({ description: "Repository-relative file or directory" }), { maxItems: 500 })),
668
+ excludePaths: Type.Optional(
669
+ Type.Array(Type.String({ description: "Repository-relative file or directory to omit" }), { maxItems: 500 }),
670
+ ),
671
+ includeIgnored: Type.Optional(
672
+ Type.Boolean({ description: "Include Git-ignored files. Default false; use only when an ignored fixture is explicitly required." }),
673
+ ),
674
+ });
675
+
676
+ export default function remoteJobsExtension(pi: ExtensionAPI) {
677
+ registerRemoteSetup(pi);
678
+
679
+ pi.registerTool({
680
+ name: "remote_status",
681
+ label: "Remote Status",
682
+ description: "Report live CPU, memory, GPU, disk, model-mode, and job-slot capacity for configured SSH workers.",
683
+ promptSnippet: "Check available remote test/build worker capacity",
684
+ parameters: Type.Object({
685
+ host: Type.Optional(Type.String({ description: "Specific worker name; omit for all workers" })),
686
+ }),
687
+ async execute(_id, params) {
688
+ const config = await loadConfig();
689
+ const enabled = config.workers.filter((worker) => worker.enabled);
690
+ if (enabled.length === 0) {
691
+ return {
692
+ content: [{ type: "text" as const, text: "No remote workers are enabled. Ask the user to run /remote setup to add or enable one." }],
693
+ details: undefined,
694
+ };
695
+ }
696
+ const selected = params.host ? enabled.filter((worker) => worker.name === params.host) : enabled;
697
+ if (params.host && selected.length === 0) throw new Error(`Unknown remote worker: ${params.host}`);
698
+ const statuses = await Promise.all(selected.map(probeWorker));
699
+ statusCache = undefined;
700
+ return {
701
+ content: [{ type: "text", text: detailedStatuses(statuses) }],
702
+ details: { statuses },
703
+ };
704
+ },
705
+ });
706
+
707
+ pi.registerTool({
708
+ name: "remote_test",
709
+ label: "Remote Test",
710
+ description:
711
+ "Snapshot the current local Git working tree (including uncommitted code), transfer the chosen files through SSH, and run a test/build command on a capacity-gated worker. The snapshot can be the full working tree, tracked files only, or explicit repository-relative paths.",
712
+ promptSnippet: "Run tests/builds on a capacity-gated SSH worker using an exact local code snapshot",
713
+ promptGuidelines: [
714
+ "Call remote_test only after relevant edits have completed; never place remote_test before edit/write calls in the same tool batch.",
715
+ "Use remote_test snapshot.mode=paths when only specific files and their manifests/lockfiles are required; otherwise use working-tree so current uncommitted code is tested.",
716
+ "When remote_test reports BLOCKED, use another READY worker or run the command locally instead of immediately retrying that worker.",
717
+ ],
718
+ executionMode: "sequential",
719
+ parameters: Type.Object({
720
+ host: Type.Optional(Type.String({ description: 'Worker name or "auto". Default: auto.' })),
721
+ command: Type.String({ minLength: 1, maxLength: 8192, description: "Test or build command to run at the snapshot root" }),
722
+ repositoryPath: Type.Optional(
723
+ Type.String({ description: "Local path inside the target Git repository. Default: current working directory." }),
724
+ ),
725
+ snapshot: Type.Optional(SnapshotSchema),
726
+ requiresGpu: Type.Optional(Type.Boolean({ description: "Reserve an exclusive GPU job slot. Default false." })),
727
+ timeoutSeconds: Type.Optional(
728
+ Type.Integer({ minimum: 10, maximum: 21600, description: "Remote command timeout. Default 1800 seconds." }),
729
+ ),
730
+ keepSource: Type.Optional(
731
+ Type.Boolean({
732
+ description:
733
+ "Keep the uploaded source tree on the worker after the run for debugging. "
734
+ + "Default false: the tree is deleted on completion and only diagnostics are retained.",
735
+ }),
736
+ ),
737
+ }),
738
+ async execute(_id, params, signal, onUpdate, ctx) {
739
+ const config = await loadConfig();
740
+ const requestedHost = params.host ?? "auto";
741
+ const enabledWorkers = config.workers.filter((worker) => worker.enabled);
742
+ if (enabledWorkers.length === 0) {
743
+ throw new Error("No remote workers are enabled. Ask the user to run /remote setup to add or enable one.");
744
+ }
745
+ if (requestedHost !== "auto" && !enabledWorkers.some((worker) => worker.name === requestedHost)) {
746
+ throw new Error(`Unknown remote worker: ${requestedHost}. Available: ${enabledWorkers.map((w) => w.name).join(", ")}`);
747
+ }
748
+ let statuses = await probeWorkers(config, true);
749
+ let worker = pickWorker(config, statuses, requestedHost, params.requiresGpu ?? false);
750
+ if (!worker) return blockedResult(undefined, undefined, detailedStatuses(statuses));
751
+ let workerStatus = statuses.find((status) => status.name === worker!.name);
752
+ if (!workerStatus || workerStatus.state !== "ready") return blockedResult(worker, workerStatus);
753
+
754
+ onUpdate?.({ content: [{ type: "text", text: `Creating selected snapshot for ${worker.name}...` }], details: { state: "snapshotting", worker: worker.name } });
755
+ const snapshot = await createSnapshot(ctx.cwd, params);
756
+ const jobId = `${new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14)}-${randomBytes(3).toString("hex")}`;
757
+ let reserved = false;
758
+ const started = Date.now();
759
+ try {
760
+ workerStatus = await probeWorker(worker);
761
+ if (workerStatus.state !== "ready") return blockedResult(worker, workerStatus);
762
+ const reservation = await reserveWorker(worker, jobId, params.requiresGpu ?? false);
763
+ if (!reservation.admitted) return blockedResult(worker, workerStatus, reservation.reason);
764
+ reserved = true;
765
+ onUpdate?.({
766
+ content: [
767
+ {
768
+ type: "text",
769
+ text: `Uploading ${snapshot.files.length} files (${formatSize(snapshot.archiveBytes)}) to ${worker.name}...`,
770
+ },
771
+ ],
772
+ details: { state: "uploading", worker: worker.name, jobId, snapshot: snapshot.fingerprint },
773
+ });
774
+ const upload = await runSsh(worker, uploadCommand(worker, snapshot.repoName, jobId), {
775
+ input: { file: snapshot.archivePath },
776
+ timeoutSeconds: Math.max(120, Math.min(900, Math.ceil(snapshot.archiveBytes / 250000) + 60)),
777
+ signal,
778
+ });
779
+ if (upload.code !== 0 || upload.timedOut || upload.aborted) {
780
+ throw new Error(upload.timedOut ? "snapshot upload timed out" : upload.stderr.trim() || `snapshot upload exited ${upload.code}`);
781
+ }
782
+ let streamed = "";
783
+ let lastUpdate = 0;
784
+ const run = await runSsh(worker, "bash -s", {
785
+ input: testScript(worker, snapshot.repoName, jobId, params.command, params.keepSource === true),
786
+ timeoutSeconds: params.timeoutSeconds ?? 1800,
787
+ signal,
788
+ onData: (chunk) => {
789
+ streamed = appendTail(streamed, chunk);
790
+ if (Date.now() - lastUpdate > 500) {
791
+ lastUpdate = Date.now();
792
+ const preview = truncateTail(streamed, { maxLines: 40, maxBytes: 8000 }).content;
793
+ onUpdate?.({
794
+ content: [{ type: "text", text: preview || `Running on ${worker!.name}...` }],
795
+ details: { state: "running", worker: worker!.name, jobId },
796
+ });
797
+ }
798
+ },
799
+ });
800
+ const durationSeconds = Math.round((Date.now() - started) / 100) / 10;
801
+ const combined = [run.stdout, run.stderr].filter(Boolean).join("\n");
802
+ const truncated = truncateTail(combined, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
803
+ const remoteJob = `${worker.root}/${snapshot.repoName}/${jobId}`;
804
+ const status = run.aborted ? "ABORTED" : run.timedOut ? "TIMED OUT" : run.code === 0 ? "PASSED" : "FAILED";
805
+ let text = [
806
+ `REMOTE TEST ${status}`,
807
+ `Worker: ${worker.name}`,
808
+ `Job: ${remoteJob}`,
809
+ `Snapshot: ${snapshot.fingerprint.slice(0, 16)} (${snapshot.mode}, ${snapshot.files.length} files, ${formatSize(snapshot.archiveBytes)})`,
810
+ `Git: ${snapshot.commit.slice(0, 12)}${snapshot.dirty ? " + working-tree changes" : ""}`,
811
+ `Command: ${params.command}`,
812
+ `Exit code: ${run.code}`,
813
+ `Duration: ${durationSeconds}s`,
814
+ "",
815
+ truncated.content || "(no output)",
816
+ ].join("\n");
817
+ if (truncated.truncated || run.totalOutputBytes > DEFAULT_MAX_BYTES) {
818
+ text += `\n\n[Output truncated; full log: ${remoteJob}/test.log]`;
819
+ }
820
+ void runSsh(worker, "bash -s", { input: cleanupScript(worker, snapshot.repoName), timeoutSeconds: 15 }).catch(() => {});
821
+ statusCache = undefined;
822
+ return {
823
+ content: [{ type: "text", text }],
824
+ details: {
825
+ state: status.toLowerCase().replace(" ", "_"),
826
+ worker: worker.name,
827
+ jobId,
828
+ remoteJob,
829
+ exitCode: run.code,
830
+ durationSeconds,
831
+ timedOut: run.timedOut,
832
+ aborted: run.aborted,
833
+ snapshot: {
834
+ mode: snapshot.mode,
835
+ fingerprint: snapshot.fingerprint,
836
+ commit: snapshot.commit,
837
+ dirty: snapshot.dirty,
838
+ fileCount: snapshot.files.length,
839
+ files: snapshot.files,
840
+ },
841
+ },
842
+ };
843
+ } finally {
844
+ await rm(snapshot.tempDir, { recursive: true, force: true });
845
+ if (reserved) await releaseWorker(worker, jobId);
846
+ }
847
+ },
848
+ });
849
+
850
+ pi.on("before_agent_start", async (event) => {
851
+ try {
852
+ const config = await loadConfig();
853
+ if (!config.injectStatus) return;
854
+ const statuses = await probeWorkers(config);
855
+ if (statuses.length === 0) return;
856
+ const line = statuses.map(compactStatus).join("; ");
857
+ return {
858
+ systemPrompt:
859
+ event.systemPrompt +
860
+ `\n\nRemote worker capacity (recent sample): ${line}. remote_test always performs a fresh hard admission check. Never retry a BLOCKED worker immediately; choose another READY worker or run locally.`,
861
+ };
862
+ } catch {
863
+ return;
864
+ }
865
+ });
866
+ }