@cabane/companion 0.6.194 → 0.6.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +154 -21
- package/dist/runtime.js +88 -7
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -7214,6 +7214,20 @@ async function reapTurnScopes(turnId) {
|
|
|
7214
7214
|
return classifyStopError(message2) === "absent" ? { failures: [] } : { failures: [message2.slice(0, 200)] };
|
|
7215
7215
|
}
|
|
7216
7216
|
}
|
|
7217
|
+
async function turnScopeOomKilled(turnId) {
|
|
7218
|
+
if (process.platform !== "linux") return false;
|
|
7219
|
+
const safe2 = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
|
|
7220
|
+
try {
|
|
7221
|
+
const { stdout } = await execFileAsync(
|
|
7222
|
+
"systemctl",
|
|
7223
|
+
["--user", "show", "--property=Result", "--value", `*-turn-${safe2}-*.scope`],
|
|
7224
|
+
{ timeout: OOM_READ_TIMEOUT_MS }
|
|
7225
|
+
);
|
|
7226
|
+
return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
|
|
7227
|
+
} catch {
|
|
7228
|
+
return false;
|
|
7229
|
+
}
|
|
7230
|
+
}
|
|
7217
7231
|
function scopeCapabilityUsable(env, timeoutMs = CAPABILITY_PROBE_TIMEOUT_MS) {
|
|
7218
7232
|
if (process.platform !== "linux") return false;
|
|
7219
7233
|
try {
|
|
@@ -7462,7 +7476,7 @@ function createTurnContainment(turnId, deps = {}) {
|
|
|
7462
7476
|
}
|
|
7463
7477
|
};
|
|
7464
7478
|
}
|
|
7465
|
-
var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
|
|
7479
|
+
var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, OOM_READ_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
|
|
7466
7480
|
var init_turn_containment = __esm({
|
|
7467
7481
|
"src/turn-containment.ts"() {
|
|
7468
7482
|
"use strict";
|
|
@@ -7471,6 +7485,7 @@ var init_turn_containment = __esm({
|
|
|
7471
7485
|
CONTAINMENT_STOP_GRACE_SEC = 5;
|
|
7472
7486
|
FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
|
|
7473
7487
|
CAPABILITY_PROBE_TIMEOUT_MS = 4e3;
|
|
7488
|
+
OOM_READ_TIMEOUT_MS = 1500;
|
|
7474
7489
|
sleep2 = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
7475
7490
|
MANAGER_ENV_KEYS = ["DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR", "PATH"];
|
|
7476
7491
|
managerEnvKey = (env) => MANAGER_ENV_KEYS.map((k) => `${k}=${env[k] ?? ""}`).join("\0");
|
|
@@ -7850,6 +7865,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log2) {
|
|
|
7850
7865
|
async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
7851
7866
|
const startedAt = Date.now();
|
|
7852
7867
|
const { turnId } = handleOpts;
|
|
7868
|
+
const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
|
|
7853
7869
|
const log2 = opts.log.child({
|
|
7854
7870
|
workspaceId: opts.workspaceId,
|
|
7855
7871
|
conversationId: payload.conversationId,
|
|
@@ -7860,7 +7876,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7860
7876
|
duties.registerAbort(turnId, controller);
|
|
7861
7877
|
const stopRunner = new AbortController();
|
|
7862
7878
|
let ready = false;
|
|
7879
|
+
let child;
|
|
7880
|
+
let stopAnnounced = false;
|
|
7881
|
+
const announceStop = () => {
|
|
7882
|
+
if (stopAnnounced || !child?.stdin.writable) return;
|
|
7883
|
+
stopAnnounced = true;
|
|
7884
|
+
child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
|
|
7885
|
+
};
|
|
7863
7886
|
const forwardStop = () => {
|
|
7887
|
+
announceStop();
|
|
7864
7888
|
if (ready) stopRunner.abort();
|
|
7865
7889
|
};
|
|
7866
7890
|
controller.signal.addEventListener("abort", forwardStop, { once: true });
|
|
@@ -7883,7 +7907,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7883
7907
|
let thrown;
|
|
7884
7908
|
let code;
|
|
7885
7909
|
let handle;
|
|
7886
|
-
let child;
|
|
7887
7910
|
const groups = [];
|
|
7888
7911
|
const remote = [];
|
|
7889
7912
|
const lockOwner = (id) => `${turnId}#${id}`;
|
|
@@ -7931,6 +7954,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7931
7954
|
stdin: runnerStdin(input)
|
|
7932
7955
|
});
|
|
7933
7956
|
child = running;
|
|
7957
|
+
if (controller.signal.aborted) announceStop();
|
|
7934
7958
|
log2.debug(
|
|
7935
7959
|
{ turnId, runnerPid: running.pid, provider: launch.provider.name },
|
|
7936
7960
|
"runner: started"
|
|
@@ -7996,6 +8020,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7996
8020
|
case "remote_session":
|
|
7997
8021
|
remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
|
|
7998
8022
|
return;
|
|
8023
|
+
case "terminated":
|
|
8024
|
+
void oomKilled(turnId).then((oom) => {
|
|
8025
|
+
if (running.stdin.writable) {
|
|
8026
|
+
running.stdin.write(
|
|
8027
|
+
encodeLauncherMessage({
|
|
8028
|
+
cabaneLauncher: "cause",
|
|
8029
|
+
cause: oom ? "out_of_memory" : null
|
|
8030
|
+
})
|
|
8031
|
+
);
|
|
8032
|
+
}
|
|
8033
|
+
});
|
|
8034
|
+
return;
|
|
7999
8035
|
}
|
|
8000
8036
|
});
|
|
8001
8037
|
const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
|
|
@@ -8013,7 +8049,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
8013
8049
|
if (result) return result;
|
|
8014
8050
|
const durationMs = Date.now() - startedAt;
|
|
8015
8051
|
const stopped = controller.signal.aborted && thrown === void 0;
|
|
8016
|
-
const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
|
|
8052
|
+
const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
|
|
8017
8053
|
await child?.reap(groups);
|
|
8018
8054
|
const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
|
|
8019
8055
|
failures: [err instanceof Error ? err.message : String(err)]
|
|
@@ -9296,7 +9332,7 @@ function fingerprintSessionState(state) {
|
|
|
9296
9332
|
}
|
|
9297
9333
|
return createHash2("sha256").update(opaqueId).digest("hex").slice(0, 16);
|
|
9298
9334
|
}
|
|
9299
|
-
var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, TurnExecution;
|
|
9335
|
+
var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, RUNNER_SIGTERM, RUNNER_TERMINATED, TurnExecution;
|
|
9300
9336
|
var init_turn_execution = __esm({
|
|
9301
9337
|
"src/turn-execution.ts"() {
|
|
9302
9338
|
"use strict";
|
|
@@ -9342,6 +9378,8 @@ var init_turn_execution = __esm({
|
|
|
9342
9378
|
reason;
|
|
9343
9379
|
errorReason;
|
|
9344
9380
|
};
|
|
9381
|
+
RUNNER_SIGTERM = "runner_sigterm";
|
|
9382
|
+
RUNNER_TERMINATED = "runner_terminated";
|
|
9345
9383
|
TurnExecution = class {
|
|
9346
9384
|
constructor(opts, supervisor, payload, handleOpts = {}) {
|
|
9347
9385
|
this.opts = opts;
|
|
@@ -9451,6 +9489,16 @@ var init_turn_execution = __esm({
|
|
|
9451
9489
|
concluded(reason, errorReason) {
|
|
9452
9490
|
return new TurnConcluded(reason, errorReason);
|
|
9453
9491
|
}
|
|
9492
|
+
// A SIGTERM the launcher never announced as a Stop.
|
|
9493
|
+
terminatedFromOutside() {
|
|
9494
|
+
const { signal } = this.abortController;
|
|
9495
|
+
return signal.aborted && signal.reason === RUNNER_SIGTERM && !(this.opts.inRunner?.stopRequested?.() ?? false);
|
|
9496
|
+
}
|
|
9497
|
+
// An abort before the run: a Stop is silent, a termination nobody asked for
|
|
9498
|
+
// is reported.
|
|
9499
|
+
concludedOnAbort() {
|
|
9500
|
+
return this.terminatedFromOutside() ? this.concluded(RUNNER_TERMINATED, RUNNER_TERMINATED) : this.concluded("cancelled");
|
|
9501
|
+
}
|
|
9454
9502
|
// The arc. Phases in order; every pre-loop exit throws `TurnConcluded` and
|
|
9455
9503
|
// funnels through ONE conclusion (hand back the controller — idempotent and
|
|
9456
9504
|
// identity-checked at the supervisor — then clear the active-run flag);
|
|
@@ -9571,7 +9619,7 @@ var init_turn_execution = __esm({
|
|
|
9571
9619
|
} catch (err) {
|
|
9572
9620
|
if (this.abortController.signal.aborted) {
|
|
9573
9621
|
turnLog.debug("dispatcher: stopped while loading the turn context");
|
|
9574
|
-
throw this.
|
|
9622
|
+
throw this.concludedOnAbort();
|
|
9575
9623
|
}
|
|
9576
9624
|
const status2 = err instanceof ApiError ? err.status : 0;
|
|
9577
9625
|
if (status2 === 404) {
|
|
@@ -9633,7 +9681,7 @@ var init_turn_execution = __esm({
|
|
|
9633
9681
|
this.executionSourceId = adopted.executionSourceId;
|
|
9634
9682
|
return "adopted";
|
|
9635
9683
|
} catch (err) {
|
|
9636
|
-
if (this.abortController.signal.aborted) throw this.
|
|
9684
|
+
if (this.abortController.signal.aborted) throw this.concludedOnAbort();
|
|
9637
9685
|
if (apiErrorCode(err) === "execution_source_unavailable") return "refused";
|
|
9638
9686
|
turnLog.warn(apiErrorLogFields(err), "dispatcher: failed to adopt the turn context");
|
|
9639
9687
|
const reason = `execution_source_unavailable: ${describeError(err)}`;
|
|
@@ -10270,7 +10318,7 @@ var init_turn_execution = __esm({
|
|
|
10270
10318
|
}
|
|
10271
10319
|
if (abortController.signal.aborted) {
|
|
10272
10320
|
o.okResult = false;
|
|
10273
|
-
o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? "cancelled";
|
|
10321
|
+
o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? (this.terminatedFromOutside() ? RUNNER_TERMINATED : "cancelled");
|
|
10274
10322
|
}
|
|
10275
10323
|
const emptyResultReason = !this.skipState.skipped && classifyEmptyResult({
|
|
10276
10324
|
ok: o.okResult,
|
|
@@ -10309,7 +10357,16 @@ var init_turn_execution = __esm({
|
|
|
10309
10357
|
o.resultReason = "lease_lost";
|
|
10310
10358
|
o.okResult = false;
|
|
10311
10359
|
}
|
|
10312
|
-
|
|
10360
|
+
let terminated = this.terminatedFromOutside() && o.timeoutReason === null && !o.leaseLost;
|
|
10361
|
+
const cause = terminated ? await this.opts.inRunner?.terminationCause?.() : null;
|
|
10362
|
+
terminated &&= this.terminatedFromOutside();
|
|
10363
|
+
if (terminated) {
|
|
10364
|
+
o.resultReason = cause ?? RUNNER_TERMINATED;
|
|
10365
|
+
o.okResult = false;
|
|
10366
|
+
} else if (o.resultReason === RUNNER_TERMINATED) {
|
|
10367
|
+
o.resultReason = "cancelled";
|
|
10368
|
+
}
|
|
10369
|
+
const userCancelled = abortController.signal.aborted && o.timeoutReason === null && !o.leaseLost && !terminated;
|
|
10313
10370
|
if (o.timeoutReason !== null) {
|
|
10314
10371
|
o.resultReason = o.timeoutReason;
|
|
10315
10372
|
o.okResult = false;
|
|
@@ -10807,6 +10864,9 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
|
|
|
10807
10864
|
let next = 0;
|
|
10808
10865
|
let resolveFirst;
|
|
10809
10866
|
let gotFirst = false;
|
|
10867
|
+
let stopRequested = false;
|
|
10868
|
+
let gone = false;
|
|
10869
|
+
const causeWaiters = /* @__PURE__ */ new Set();
|
|
10810
10870
|
const first = new Promise((resolve2) => resolveFirst = resolve2);
|
|
10811
10871
|
const lines2 = createInterface3({ input: stdin, crlfDelay: Infinity });
|
|
10812
10872
|
lines2.on("line", (line) => {
|
|
@@ -10817,13 +10877,35 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
|
|
|
10817
10877
|
}
|
|
10818
10878
|
const message2 = decodeLauncherMessage(line);
|
|
10819
10879
|
if (message2?.cabaneLauncher === "granted") waiting.get(message2.id)?.();
|
|
10880
|
+
if (message2?.cabaneLauncher === "stop") stopRequested = true;
|
|
10881
|
+
if (message2?.cabaneLauncher === "cause") {
|
|
10882
|
+
for (const resolve2 of causeWaiters) resolve2(message2.cause);
|
|
10883
|
+
}
|
|
10820
10884
|
});
|
|
10821
10885
|
lines2.once("close", () => {
|
|
10886
|
+
gone = true;
|
|
10887
|
+
for (const resolve2 of causeWaiters) resolve2(null);
|
|
10822
10888
|
if (!gotFirst) resolveFirst("");
|
|
10823
10889
|
onLauncherGone();
|
|
10824
10890
|
});
|
|
10825
10891
|
return {
|
|
10826
10892
|
first,
|
|
10893
|
+
get stopRequested() {
|
|
10894
|
+
return stopRequested;
|
|
10895
|
+
},
|
|
10896
|
+
terminationCause(timeoutMs = CAUSE_TIMEOUT_MS) {
|
|
10897
|
+
if (gone) return Promise.resolve(null);
|
|
10898
|
+
return new Promise((resolve2) => {
|
|
10899
|
+
const done = (cause) => {
|
|
10900
|
+
clearTimeout(timer);
|
|
10901
|
+
causeWaiters.delete(done);
|
|
10902
|
+
resolve2(cause);
|
|
10903
|
+
};
|
|
10904
|
+
const timer = setTimeout(() => done(null), timeoutMs);
|
|
10905
|
+
causeWaiters.add(done);
|
|
10906
|
+
emit({ cabaneRunner: "terminated" });
|
|
10907
|
+
});
|
|
10908
|
+
},
|
|
10827
10909
|
acquire(key, signal) {
|
|
10828
10910
|
const id = String(++next);
|
|
10829
10911
|
return new Promise((resolve2, reject) => {
|
|
@@ -10850,10 +10932,12 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
|
|
|
10850
10932
|
}
|
|
10851
10933
|
};
|
|
10852
10934
|
}
|
|
10935
|
+
var CAUSE_TIMEOUT_MS;
|
|
10853
10936
|
var init_launcher_channel = __esm({
|
|
10854
10937
|
"src/runner/launcher-channel.ts"() {
|
|
10855
10938
|
"use strict";
|
|
10856
10939
|
init_protocol();
|
|
10940
|
+
CAUSE_TIMEOUT_MS = 2e3;
|
|
10857
10941
|
}
|
|
10858
10942
|
});
|
|
10859
10943
|
|
|
@@ -10950,6 +11034,8 @@ function runnerTurnOptions(input, log2, emit, extras = {}) {
|
|
|
10950
11034
|
(sessionId) => emit({ cabaneRunner: "remote_session", serverUrl, sessionId })
|
|
10951
11035
|
),
|
|
10952
11036
|
...extras.onProcessGroup ? { onProcessGroup: extras.onProcessGroup } : {},
|
|
11037
|
+
...extras.stopRequested ? { stopRequested: extras.stopRequested } : {},
|
|
11038
|
+
...extras.terminationCause ? { terminationCause: extras.terminationCause } : {},
|
|
10953
11039
|
codexExecutable: (path, containment) => {
|
|
10954
11040
|
const prefix = containment.scopePrefix(process.env);
|
|
10955
11041
|
return prefix ? shims.write([...prefix, path]) : path;
|
|
@@ -11007,16 +11093,16 @@ async function runTurn(env, deps) {
|
|
|
11007
11093
|
const controllers = /* @__PURE__ */ new Set();
|
|
11008
11094
|
const shims = createShimDir();
|
|
11009
11095
|
let terminated = false;
|
|
11010
|
-
const onTerm = () => {
|
|
11011
|
-
terminated = true;
|
|
11012
|
-
for (const controller of controllers) controller.abort();
|
|
11013
|
-
};
|
|
11014
|
-
deps.signals.on("SIGTERM", onTerm);
|
|
11015
11096
|
const groups = [];
|
|
11016
11097
|
const channel = openLauncherChannel(deps.stdin, emit, () => {
|
|
11017
11098
|
if (deps.onLauncherGone) deps.onLauncherGone(groups);
|
|
11018
11099
|
else void stopWithLauncher(groups);
|
|
11019
11100
|
});
|
|
11101
|
+
const onTerm = () => {
|
|
11102
|
+
terminated = true;
|
|
11103
|
+
for (const controller of controllers) controller.abort(RUNNER_SIGTERM);
|
|
11104
|
+
};
|
|
11105
|
+
deps.signals.on("SIGTERM", onTerm);
|
|
11020
11106
|
emit({ cabaneRunner: "ready" });
|
|
11021
11107
|
try {
|
|
11022
11108
|
const input = parseRunnerInput(env, await channel.first);
|
|
@@ -11026,7 +11112,7 @@ async function runTurn(env, deps) {
|
|
|
11026
11112
|
const supervisor = {
|
|
11027
11113
|
registerAbort: (_turnId, controller) => {
|
|
11028
11114
|
controllers.add(controller);
|
|
11029
|
-
if (terminated) controller.abort();
|
|
11115
|
+
if (terminated) controller.abort(RUNNER_SIGTERM);
|
|
11030
11116
|
},
|
|
11031
11117
|
releaseAbort: (_turnId, controller) => {
|
|
11032
11118
|
controllers.delete(controller);
|
|
@@ -11045,6 +11131,8 @@ async function runTurn(env, deps) {
|
|
|
11045
11131
|
...deps.queryFn ? { queryFn: deps.queryFn } : {},
|
|
11046
11132
|
shims,
|
|
11047
11133
|
acquireLock: (key, signal) => channel.acquire(key, signal),
|
|
11134
|
+
stopRequested: () => channel.stopRequested,
|
|
11135
|
+
terminationCause: () => channel.terminationCause(),
|
|
11048
11136
|
onProcessGroup: (pgid) => {
|
|
11049
11137
|
groups.push(pgid);
|
|
11050
11138
|
emit({ cabaneRunner: "process_group", pgid });
|
|
@@ -11073,6 +11161,7 @@ var init_run_turn = __esm({
|
|
|
11073
11161
|
init_outbox();
|
|
11074
11162
|
init_exec_shim();
|
|
11075
11163
|
init_launcher_channel();
|
|
11164
|
+
init_turn_execution();
|
|
11076
11165
|
init_opencode_lock();
|
|
11077
11166
|
init_secrets();
|
|
11078
11167
|
init_transcript_writer();
|
|
@@ -11287,7 +11376,6 @@ import { join as join7 } from "path";
|
|
|
11287
11376
|
// src/device-state.ts
|
|
11288
11377
|
var DEVICE_STATES = ["waiting", "online", "offline", "deactivated"];
|
|
11289
11378
|
var DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS = 9e4;
|
|
11290
|
-
var DEFAULT_DEVICE_RECONNECT_GRACE_MS = 9e4;
|
|
11291
11379
|
var HARNESS_STATES = [
|
|
11292
11380
|
"checking",
|
|
11293
11381
|
"connected",
|
|
@@ -14993,8 +15081,6 @@ var CompanionSupervisor = class {
|
|
|
14993
15081
|
const key = `${run.conversationId}|${run.agentId}`;
|
|
14994
15082
|
if (wr.chains.has(key)) continue;
|
|
14995
15083
|
this.recoveredTurns.add(run.turnId);
|
|
14996
|
-
if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
|
|
14997
|
-
continue;
|
|
14998
15084
|
const payload = {
|
|
14999
15085
|
type: "device:dispatch_requested",
|
|
15000
15086
|
deviceId: this.deviceId,
|
|
@@ -15002,6 +15088,11 @@ var CompanionSupervisor = class {
|
|
|
15002
15088
|
conversationId: run.conversationId,
|
|
15003
15089
|
messageId: run.messageId
|
|
15004
15090
|
};
|
|
15091
|
+
const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
|
|
15092
|
+
if (attempt > MAX_RESUME_ATTEMPTS) {
|
|
15093
|
+
await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
|
|
15094
|
+
continue;
|
|
15095
|
+
}
|
|
15005
15096
|
const tail = this.runDispatch(
|
|
15006
15097
|
wr,
|
|
15007
15098
|
// Off `/runs`, not a frame, so no dispatch id to dedupe on;
|
|
@@ -15027,6 +15118,44 @@ var CompanionSupervisor = class {
|
|
|
15027
15118
|
this.recovering = false;
|
|
15028
15119
|
}
|
|
15029
15120
|
}
|
|
15121
|
+
// An interrupted turn this device has tried to resume too often: it stops
|
|
15122
|
+
// trying, and the server is told the turn failed, so the conversation says it
|
|
15123
|
+
// couldn't resume instead of going quiet. The settle the launcher sends for a
|
|
15124
|
+
// turn it couldn't run: it ends the span under this id if one is still
|
|
15125
|
+
// running, disposes of the offer if none ever started, and is a no-op on a
|
|
15126
|
+
// turn that already ended. A settle that doesn't land leaves the turn to the
|
|
15127
|
+
// server's sweep, as before.
|
|
15128
|
+
async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
|
|
15129
|
+
this.log.error(
|
|
15130
|
+
{
|
|
15131
|
+
workspaceId,
|
|
15132
|
+
conversationId: payload.conversationId,
|
|
15133
|
+
agentId: payload.agentId,
|
|
15134
|
+
turnId,
|
|
15135
|
+
...eventId ? { eventId } : {},
|
|
15136
|
+
attempt,
|
|
15137
|
+
cap: MAX_RESUME_ATTEMPTS
|
|
15138
|
+
},
|
|
15139
|
+
"An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
|
|
15140
|
+
);
|
|
15141
|
+
try {
|
|
15142
|
+
await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
|
|
15143
|
+
activeRunStartedAt: null,
|
|
15144
|
+
turnId,
|
|
15145
|
+
outcome: "failed",
|
|
15146
|
+
errorReason: "resume_exhausted",
|
|
15147
|
+
readThrough: payload.messageId,
|
|
15148
|
+
// The delivery being given up on, so a newer one of the same message
|
|
15149
|
+
// is not failed in its name (the server checks under its lock).
|
|
15150
|
+
...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
|
|
15151
|
+
});
|
|
15152
|
+
} catch (err) {
|
|
15153
|
+
this.log.warn(
|
|
15154
|
+
{ ...apiErrorLogFields(err), turnId },
|
|
15155
|
+
"Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
|
|
15156
|
+
);
|
|
15157
|
+
}
|
|
15158
|
+
}
|
|
15030
15159
|
// One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
|
|
15031
15160
|
async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
|
|
15032
15161
|
await this.initialHarnessReady;
|
|
@@ -15050,9 +15179,13 @@ var CompanionSupervisor = class {
|
|
|
15050
15179
|
}
|
|
15051
15180
|
const attempt = bumpResumeAttempt(workspaceId, dispatchId);
|
|
15052
15181
|
if (attempt > MAX_RESUME_ATTEMPTS) {
|
|
15053
|
-
this.
|
|
15054
|
-
|
|
15055
|
-
|
|
15182
|
+
await this.abandonResume(
|
|
15183
|
+
workspaceId,
|
|
15184
|
+
agent,
|
|
15185
|
+
payload,
|
|
15186
|
+
turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
|
|
15187
|
+
attempt,
|
|
15188
|
+
dispatchId
|
|
15056
15189
|
);
|
|
15057
15190
|
markCompleted(workspaceId, dispatchId);
|
|
15058
15191
|
return;
|
|
@@ -16246,7 +16379,7 @@ function defaultSpawnDetached(args) {
|
|
|
16246
16379
|
// src/terminal-copy.ts
|
|
16247
16380
|
init_cabane_command();
|
|
16248
16381
|
var UNPAIRED = "This device isn't paired yet. Get a setup line from a workspace's Settings \u2192 Devices and paste it here.";
|
|
16249
|
-
var lease = DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS
|
|
16382
|
+
var lease = DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS;
|
|
16250
16383
|
function age(at, now) {
|
|
16251
16384
|
const seconds = Math.max(0, Math.floor((now - Date.parse(at)) / 1e3));
|
|
16252
16385
|
return seconds < 60 ? `${seconds}s` : seconds < 3600 ? `${Math.floor(seconds / 60)}m` : `${Math.floor(seconds / 3600)}h`;
|
package/dist/runtime.js
CHANGED
|
@@ -7340,6 +7340,7 @@ import { promisify } from "util";
|
|
|
7340
7340
|
var execFileAsync = promisify(execFile);
|
|
7341
7341
|
var CONTAINMENT_STOP_GRACE_SEC = 5;
|
|
7342
7342
|
var FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
|
|
7343
|
+
var OOM_READ_TIMEOUT_MS = 1500;
|
|
7343
7344
|
function turnScopeUnit(turnId, attempt) {
|
|
7344
7345
|
const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
|
|
7345
7346
|
return `cabane-turn-${safe}-${attempt}`;
|
|
@@ -7357,6 +7358,20 @@ async function reapTurnScopes(turnId) {
|
|
|
7357
7358
|
return classifyStopError(message) === "absent" ? { failures: [] } : { failures: [message.slice(0, 200)] };
|
|
7358
7359
|
}
|
|
7359
7360
|
}
|
|
7361
|
+
async function turnScopeOomKilled(turnId) {
|
|
7362
|
+
if (process.platform !== "linux") return false;
|
|
7363
|
+
const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
|
|
7364
|
+
try {
|
|
7365
|
+
const { stdout } = await execFileAsync(
|
|
7366
|
+
"systemctl",
|
|
7367
|
+
["--user", "show", "--property=Result", "--value", `*-turn-${safe}-*.scope`],
|
|
7368
|
+
{ timeout: OOM_READ_TIMEOUT_MS }
|
|
7369
|
+
);
|
|
7370
|
+
return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
|
|
7371
|
+
} catch {
|
|
7372
|
+
return false;
|
|
7373
|
+
}
|
|
7374
|
+
}
|
|
7360
7375
|
function classifyStopError(message) {
|
|
7361
7376
|
if (/failed to connect to bus|failed to get d-?bus connection|connection refused|refusing to operate|spawn systemctl|systemctl.*ENOENT|ENOENT.*systemctl/i.test(
|
|
7362
7377
|
message
|
|
@@ -7697,6 +7712,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log) {
|
|
|
7697
7712
|
async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
7698
7713
|
const startedAt = Date.now();
|
|
7699
7714
|
const { turnId } = handleOpts;
|
|
7715
|
+
const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
|
|
7700
7716
|
const log = opts.log.child({
|
|
7701
7717
|
workspaceId: opts.workspaceId,
|
|
7702
7718
|
conversationId: payload.conversationId,
|
|
@@ -7707,7 +7723,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7707
7723
|
duties.registerAbort(turnId, controller);
|
|
7708
7724
|
const stopRunner = new AbortController();
|
|
7709
7725
|
let ready = false;
|
|
7726
|
+
let child;
|
|
7727
|
+
let stopAnnounced = false;
|
|
7728
|
+
const announceStop = () => {
|
|
7729
|
+
if (stopAnnounced || !child?.stdin.writable) return;
|
|
7730
|
+
stopAnnounced = true;
|
|
7731
|
+
child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
|
|
7732
|
+
};
|
|
7710
7733
|
const forwardStop = () => {
|
|
7734
|
+
announceStop();
|
|
7711
7735
|
if (ready) stopRunner.abort();
|
|
7712
7736
|
};
|
|
7713
7737
|
controller.signal.addEventListener("abort", forwardStop, { once: true });
|
|
@@ -7730,7 +7754,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7730
7754
|
let thrown;
|
|
7731
7755
|
let code;
|
|
7732
7756
|
let handle;
|
|
7733
|
-
let child;
|
|
7734
7757
|
const groups = [];
|
|
7735
7758
|
const remote = [];
|
|
7736
7759
|
const lockOwner = (id) => `${turnId}#${id}`;
|
|
@@ -7778,6 +7801,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7778
7801
|
stdin: runnerStdin(input)
|
|
7779
7802
|
});
|
|
7780
7803
|
child = running;
|
|
7804
|
+
if (controller.signal.aborted) announceStop();
|
|
7781
7805
|
log.debug(
|
|
7782
7806
|
{ turnId, runnerPid: running.pid, provider: launch.provider.name },
|
|
7783
7807
|
"runner: started"
|
|
@@ -7843,6 +7867,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7843
7867
|
case "remote_session":
|
|
7844
7868
|
remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
|
|
7845
7869
|
return;
|
|
7870
|
+
case "terminated":
|
|
7871
|
+
void oomKilled(turnId).then((oom) => {
|
|
7872
|
+
if (running.stdin.writable) {
|
|
7873
|
+
running.stdin.write(
|
|
7874
|
+
encodeLauncherMessage({
|
|
7875
|
+
cabaneLauncher: "cause",
|
|
7876
|
+
cause: oom ? "out_of_memory" : null
|
|
7877
|
+
})
|
|
7878
|
+
);
|
|
7879
|
+
}
|
|
7880
|
+
});
|
|
7881
|
+
return;
|
|
7846
7882
|
}
|
|
7847
7883
|
});
|
|
7848
7884
|
const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
|
|
@@ -7860,7 +7896,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
|
|
|
7860
7896
|
if (result) return result;
|
|
7861
7897
|
const durationMs = Date.now() - startedAt;
|
|
7862
7898
|
const stopped = controller.signal.aborted && thrown === void 0;
|
|
7863
|
-
const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
|
|
7899
|
+
const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
|
|
7864
7900
|
await child?.reap(groups);
|
|
7865
7901
|
const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
|
|
7866
7902
|
failures: [err instanceof Error ? err.message : String(err)]
|
|
@@ -10655,8 +10691,6 @@ var CompanionSupervisor = class {
|
|
|
10655
10691
|
const key = `${run.conversationId}|${run.agentId}`;
|
|
10656
10692
|
if (wr.chains.has(key)) continue;
|
|
10657
10693
|
this.recoveredTurns.add(run.turnId);
|
|
10658
|
-
if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
|
|
10659
|
-
continue;
|
|
10660
10694
|
const payload = {
|
|
10661
10695
|
type: "device:dispatch_requested",
|
|
10662
10696
|
deviceId: this.deviceId,
|
|
@@ -10664,6 +10698,11 @@ var CompanionSupervisor = class {
|
|
|
10664
10698
|
conversationId: run.conversationId,
|
|
10665
10699
|
messageId: run.messageId
|
|
10666
10700
|
};
|
|
10701
|
+
const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
|
|
10702
|
+
if (attempt > MAX_RESUME_ATTEMPTS) {
|
|
10703
|
+
await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
|
|
10704
|
+
continue;
|
|
10705
|
+
}
|
|
10667
10706
|
const tail = this.runDispatch(
|
|
10668
10707
|
wr,
|
|
10669
10708
|
// Off `/runs`, not a frame, so no dispatch id to dedupe on;
|
|
@@ -10689,6 +10728,44 @@ var CompanionSupervisor = class {
|
|
|
10689
10728
|
this.recovering = false;
|
|
10690
10729
|
}
|
|
10691
10730
|
}
|
|
10731
|
+
// An interrupted turn this device has tried to resume too often: it stops
|
|
10732
|
+
// trying, and the server is told the turn failed, so the conversation says it
|
|
10733
|
+
// couldn't resume instead of going quiet. The settle the launcher sends for a
|
|
10734
|
+
// turn it couldn't run: it ends the span under this id if one is still
|
|
10735
|
+
// running, disposes of the offer if none ever started, and is a no-op on a
|
|
10736
|
+
// turn that already ended. A settle that doesn't land leaves the turn to the
|
|
10737
|
+
// server's sweep, as before.
|
|
10738
|
+
async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
|
|
10739
|
+
this.log.error(
|
|
10740
|
+
{
|
|
10741
|
+
workspaceId,
|
|
10742
|
+
conversationId: payload.conversationId,
|
|
10743
|
+
agentId: payload.agentId,
|
|
10744
|
+
turnId,
|
|
10745
|
+
...eventId ? { eventId } : {},
|
|
10746
|
+
attempt,
|
|
10747
|
+
cap: MAX_RESUME_ATTEMPTS
|
|
10748
|
+
},
|
|
10749
|
+
"An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
|
|
10750
|
+
);
|
|
10751
|
+
try {
|
|
10752
|
+
await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
|
|
10753
|
+
activeRunStartedAt: null,
|
|
10754
|
+
turnId,
|
|
10755
|
+
outcome: "failed",
|
|
10756
|
+
errorReason: "resume_exhausted",
|
|
10757
|
+
readThrough: payload.messageId,
|
|
10758
|
+
// The delivery being given up on, so a newer one of the same message
|
|
10759
|
+
// is not failed in its name (the server checks under its lock).
|
|
10760
|
+
...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
|
|
10761
|
+
});
|
|
10762
|
+
} catch (err) {
|
|
10763
|
+
this.log.warn(
|
|
10764
|
+
{ ...apiErrorLogFields(err), turnId },
|
|
10765
|
+
"Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
|
|
10766
|
+
);
|
|
10767
|
+
}
|
|
10768
|
+
}
|
|
10692
10769
|
// One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
|
|
10693
10770
|
async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
|
|
10694
10771
|
await this.initialHarnessReady;
|
|
@@ -10712,9 +10789,13 @@ var CompanionSupervisor = class {
|
|
|
10712
10789
|
}
|
|
10713
10790
|
const attempt = bumpResumeAttempt(workspaceId, dispatchId);
|
|
10714
10791
|
if (attempt > MAX_RESUME_ATTEMPTS) {
|
|
10715
|
-
this.
|
|
10716
|
-
|
|
10717
|
-
|
|
10792
|
+
await this.abandonResume(
|
|
10793
|
+
workspaceId,
|
|
10794
|
+
agent,
|
|
10795
|
+
payload,
|
|
10796
|
+
turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
|
|
10797
|
+
attempt,
|
|
10798
|
+
dispatchId
|
|
10718
10799
|
);
|
|
10719
10800
|
markCompleted(workspaceId, dispatchId);
|
|
10720
10801
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cabane/companion",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.196",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The Cabane Companion (headless): connect an agent on your machine to your Cabane workspace as a responder — drive work against your own codebase, files, and MCP servers without putting any of it in Cabane.",
|
|
6
6
|
"license": "UNLICENSED",
|