@runuai/host 0.9.86 → 0.9.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agents/machine-durable.ts +61 -3
- package/package.json +1 -1
- package/src/index.ts +7 -12
|
@@ -143,6 +143,8 @@ export class MachineDurableProcess implements LineTransport {
|
|
|
143
143
|
#closed = false;
|
|
144
144
|
#detached = false;
|
|
145
145
|
#sawSpawnMeta = false;
|
|
146
|
+
/** Completed-and-empty direct outbox probes past the spawn grace. */
|
|
147
|
+
#noMetaStrikes = 0;
|
|
146
148
|
readonly #startedAt = Date.now();
|
|
147
149
|
#tail: TaskEnvironmentProcess | null = null;
|
|
148
150
|
#tailGeneration = 0;
|
|
@@ -322,9 +324,40 @@ export class MachineDurableProcess implements LineTransport {
|
|
|
322
324
|
this.#probing = true;
|
|
323
325
|
try {
|
|
324
326
|
if (!this.#sawSpawnMeta) {
|
|
325
|
-
if (Date.now() - this.#startedAt
|
|
326
|
-
|
|
327
|
+
if (Date.now() - this.#startedAt <= HEARTBEAT_STALE_MS) return;
|
|
328
|
+
// Grace elapsed on OUR clock — but the spawn meta arrives via the
|
|
329
|
+
// tail, an ssh child of THIS host, and both starve together when the
|
|
330
|
+
// host CPU is pegged (live 2026-08-29: a 12-task launch burst pushed
|
|
331
|
+
// load to 16, tails lagged, and healthy runners were declared dead —
|
|
332
|
+
// then respawned, ORPHANING the originals). Before any verdict, ask
|
|
333
|
+
// the machine directly: the runner writes its spawn meta as the
|
|
334
|
+
// outbox's first line.
|
|
335
|
+
let sawOnMachine = false;
|
|
336
|
+
try {
|
|
337
|
+
const probe = await execStep(this.#environment, [
|
|
338
|
+
"/bin/sh",
|
|
339
|
+
"-c",
|
|
340
|
+
'head -n1 "$1/outbox.jsonl" 2>/dev/null | grep -c __uai',
|
|
341
|
+
"probe",
|
|
342
|
+
this.#sessionDir,
|
|
343
|
+
]);
|
|
344
|
+
sawOnMachine =
|
|
345
|
+
probe.exitCode === 0 && Number(probe.stdout.trim()) > 0;
|
|
346
|
+
} catch {
|
|
347
|
+
// Unreachable machine: counts as a strike below.
|
|
348
|
+
}
|
|
349
|
+
if (this.#closed || this.#detached) return;
|
|
350
|
+
if (sawOnMachine) {
|
|
351
|
+
this.#sawSpawnMeta = true;
|
|
352
|
+
return;
|
|
327
353
|
}
|
|
354
|
+
// Two COMPLETED empty probes past the grace — not elapsed time — are
|
|
355
|
+
// the death sentence; probes that never ran prove nothing.
|
|
356
|
+
this.#noMetaStrikes += 1;
|
|
357
|
+
if (this.#noMetaStrikes < 2) return;
|
|
358
|
+
await this.#killRemoteRunner();
|
|
359
|
+
if (this.#closed || this.#detached) return;
|
|
360
|
+
this.#finish(null, "no spawn meta within grace");
|
|
328
361
|
return;
|
|
329
362
|
}
|
|
330
363
|
const fresh = await machineHeartbeatFresh(
|
|
@@ -333,12 +366,37 @@ export class MachineDurableProcess implements LineTransport {
|
|
|
333
366
|
HEARTBEAT_STALE_MS,
|
|
334
367
|
);
|
|
335
368
|
if (this.#closed || this.#detached) return;
|
|
336
|
-
if (!fresh)
|
|
369
|
+
if (!fresh) {
|
|
370
|
+
// A stale-but-alive runner would be doubled by the respawn; the
|
|
371
|
+
// single-runner-per-session invariant is enforced here, not hoped for.
|
|
372
|
+
await this.#killRemoteRunner();
|
|
373
|
+
if (this.#closed || this.#detached) return;
|
|
374
|
+
this.#finish(null, "heartbeat stale");
|
|
375
|
+
}
|
|
337
376
|
} finally {
|
|
338
377
|
this.#probing = false;
|
|
339
378
|
}
|
|
340
379
|
}
|
|
341
380
|
|
|
381
|
+
/** Best-effort remote kill of this session's runner before declaring it
|
|
382
|
+
* dead: a wrong verdict must cost a restart, never an orphaned agent.
|
|
383
|
+
* `runner[.]mjs` so the kill-shell's own argv (which carries this script
|
|
384
|
+
* text) can never self-match — the pgrep/pkill self-match class. */
|
|
385
|
+
async #killRemoteRunner(): Promise<void> {
|
|
386
|
+
try {
|
|
387
|
+
await execStep(this.#environment, [
|
|
388
|
+
"/bin/sh",
|
|
389
|
+
"-c",
|
|
390
|
+
'pkill -f "runner[.]mjs $1" >/dev/null 2>&1; true',
|
|
391
|
+
"kill",
|
|
392
|
+
this.#sessionDir,
|
|
393
|
+
]);
|
|
394
|
+
} catch {
|
|
395
|
+
// The finish verdict stands either way; the orphan sweep in recovery
|
|
396
|
+
// remains the backstop for an unreachable machine.
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
342
400
|
#finish(code: number | null, reason = "unspecified"): void {
|
|
343
401
|
if (this.#closed) return;
|
|
344
402
|
// Session deaths are load-bearing and were undiagnosable without this
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -976,18 +976,13 @@ export const hostCommands: HostCommands = {
|
|
|
976
976
|
}
|
|
977
977
|
const result = await wrapAgent(ctx, "taskDown", async () => {
|
|
978
978
|
if (environment === null) return agent.taskDown(input);
|
|
979
|
-
//
|
|
980
|
-
//
|
|
981
|
-
//
|
|
982
|
-
//
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
) {
|
|
987
|
-
stopMachineGatewayTunnel(input.taskId);
|
|
988
|
-
await environment.stop();
|
|
989
|
-
return taskDownResultForInput(input, { status: "stopped" });
|
|
990
|
-
}
|
|
979
|
+
// taskDown is TERMINAL — its only cloud callers are task close and
|
|
980
|
+
// failed-occurrence/orphan cleanup; pause rides the separate stopTask
|
|
981
|
+
// command. Machines therefore TERMINATE here: a closed task can never
|
|
982
|
+
// be resumed, so a stopped-but-kept machine is a per-month disk leak
|
|
983
|
+
// with no consumer (owner, 2026-08-29 — 16 parked disks after one
|
|
984
|
+
// evening of testing). The task branch lives on the remote; the
|
|
985
|
+
// machine's disk is not a preservation surface once the task ends.
|
|
991
986
|
if (environment.descriptor.locator.provider === "machine") {
|
|
992
987
|
stopMachineGatewayTunnel(input.taskId);
|
|
993
988
|
}
|