@cabane/companion 0.6.194 → 0.6.195

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -7214,6 +7214,20 @@ async function reapTurnScopes(turnId) {
7214
7214
  return classifyStopError(message2) === "absent" ? { failures: [] } : { failures: [message2.slice(0, 200)] };
7215
7215
  }
7216
7216
  }
7217
+ async function turnScopeOomKilled(turnId) {
7218
+ if (process.platform !== "linux") return false;
7219
+ const safe2 = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7220
+ try {
7221
+ const { stdout } = await execFileAsync(
7222
+ "systemctl",
7223
+ ["--user", "show", "--property=Result", "--value", `*-turn-${safe2}-*.scope`],
7224
+ { timeout: OOM_READ_TIMEOUT_MS }
7225
+ );
7226
+ return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
7227
+ } catch {
7228
+ return false;
7229
+ }
7230
+ }
7217
7231
  function scopeCapabilityUsable(env, timeoutMs = CAPABILITY_PROBE_TIMEOUT_MS) {
7218
7232
  if (process.platform !== "linux") return false;
7219
7233
  try {
@@ -7462,7 +7476,7 @@ function createTurnContainment(turnId, deps = {}) {
7462
7476
  }
7463
7477
  };
7464
7478
  }
7465
- var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
7479
+ var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, OOM_READ_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
7466
7480
  var init_turn_containment = __esm({
7467
7481
  "src/turn-containment.ts"() {
7468
7482
  "use strict";
@@ -7471,6 +7485,7 @@ var init_turn_containment = __esm({
7471
7485
  CONTAINMENT_STOP_GRACE_SEC = 5;
7472
7486
  FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
7473
7487
  CAPABILITY_PROBE_TIMEOUT_MS = 4e3;
7488
+ OOM_READ_TIMEOUT_MS = 1500;
7474
7489
  sleep2 = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
7475
7490
  MANAGER_ENV_KEYS = ["DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR", "PATH"];
7476
7491
  managerEnvKey = (env) => MANAGER_ENV_KEYS.map((k) => `${k}=${env[k] ?? ""}`).join("\0");
@@ -7850,6 +7865,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log2) {
7850
7865
  async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7851
7866
  const startedAt = Date.now();
7852
7867
  const { turnId } = handleOpts;
7868
+ const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
7853
7869
  const log2 = opts.log.child({
7854
7870
  workspaceId: opts.workspaceId,
7855
7871
  conversationId: payload.conversationId,
@@ -7860,7 +7876,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7860
7876
  duties.registerAbort(turnId, controller);
7861
7877
  const stopRunner = new AbortController();
7862
7878
  let ready = false;
7879
+ let child;
7880
+ let stopAnnounced = false;
7881
+ const announceStop = () => {
7882
+ if (stopAnnounced || !child?.stdin.writable) return;
7883
+ stopAnnounced = true;
7884
+ child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
7885
+ };
7863
7886
  const forwardStop = () => {
7887
+ announceStop();
7864
7888
  if (ready) stopRunner.abort();
7865
7889
  };
7866
7890
  controller.signal.addEventListener("abort", forwardStop, { once: true });
@@ -7883,7 +7907,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7883
7907
  let thrown;
7884
7908
  let code;
7885
7909
  let handle;
7886
- let child;
7887
7910
  const groups = [];
7888
7911
  const remote = [];
7889
7912
  const lockOwner = (id) => `${turnId}#${id}`;
@@ -7931,6 +7954,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7931
7954
  stdin: runnerStdin(input)
7932
7955
  });
7933
7956
  child = running;
7957
+ if (controller.signal.aborted) announceStop();
7934
7958
  log2.debug(
7935
7959
  { turnId, runnerPid: running.pid, provider: launch.provider.name },
7936
7960
  "runner: started"
@@ -7996,6 +8020,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7996
8020
  case "remote_session":
7997
8021
  remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
7998
8022
  return;
8023
+ case "terminated":
8024
+ void oomKilled(turnId).then((oom) => {
8025
+ if (running.stdin.writable) {
8026
+ running.stdin.write(
8027
+ encodeLauncherMessage({
8028
+ cabaneLauncher: "cause",
8029
+ cause: oom ? "out_of_memory" : null
8030
+ })
8031
+ );
8032
+ }
8033
+ });
8034
+ return;
7999
8035
  }
8000
8036
  });
8001
8037
  const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
@@ -8013,7 +8049,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
8013
8049
  if (result) return result;
8014
8050
  const durationMs = Date.now() - startedAt;
8015
8051
  const stopped = controller.signal.aborted && thrown === void 0;
8016
- const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
8052
+ const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
8017
8053
  await child?.reap(groups);
8018
8054
  const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
8019
8055
  failures: [err instanceof Error ? err.message : String(err)]
@@ -9296,7 +9332,7 @@ function fingerprintSessionState(state) {
9296
9332
  }
9297
9333
  return createHash2("sha256").update(opaqueId).digest("hex").slice(0, 16);
9298
9334
  }
9299
- var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, TurnExecution;
9335
+ var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, RUNNER_SIGTERM, RUNNER_TERMINATED, TurnExecution;
9300
9336
  var init_turn_execution = __esm({
9301
9337
  "src/turn-execution.ts"() {
9302
9338
  "use strict";
@@ -9342,6 +9378,8 @@ var init_turn_execution = __esm({
9342
9378
  reason;
9343
9379
  errorReason;
9344
9380
  };
9381
+ RUNNER_SIGTERM = "runner_sigterm";
9382
+ RUNNER_TERMINATED = "runner_terminated";
9345
9383
  TurnExecution = class {
9346
9384
  constructor(opts, supervisor, payload, handleOpts = {}) {
9347
9385
  this.opts = opts;
@@ -9451,6 +9489,16 @@ var init_turn_execution = __esm({
9451
9489
  concluded(reason, errorReason) {
9452
9490
  return new TurnConcluded(reason, errorReason);
9453
9491
  }
9492
+ // A SIGTERM the launcher never announced as a Stop.
9493
+ terminatedFromOutside() {
9494
+ const { signal } = this.abortController;
9495
+ return signal.aborted && signal.reason === RUNNER_SIGTERM && !(this.opts.inRunner?.stopRequested?.() ?? false);
9496
+ }
9497
+ // An abort before the run: a Stop is silent, a termination nobody asked for
9498
+ // is reported.
9499
+ concludedOnAbort() {
9500
+ return this.terminatedFromOutside() ? this.concluded(RUNNER_TERMINATED, RUNNER_TERMINATED) : this.concluded("cancelled");
9501
+ }
9454
9502
  // The arc. Phases in order; every pre-loop exit throws `TurnConcluded` and
9455
9503
  // funnels through ONE conclusion (hand back the controller — idempotent and
9456
9504
  // identity-checked at the supervisor — then clear the active-run flag);
@@ -9571,7 +9619,7 @@ var init_turn_execution = __esm({
9571
9619
  } catch (err) {
9572
9620
  if (this.abortController.signal.aborted) {
9573
9621
  turnLog.debug("dispatcher: stopped while loading the turn context");
9574
- throw this.concluded("cancelled");
9622
+ throw this.concludedOnAbort();
9575
9623
  }
9576
9624
  const status2 = err instanceof ApiError ? err.status : 0;
9577
9625
  if (status2 === 404) {
@@ -9633,7 +9681,7 @@ var init_turn_execution = __esm({
9633
9681
  this.executionSourceId = adopted.executionSourceId;
9634
9682
  return "adopted";
9635
9683
  } catch (err) {
9636
- if (this.abortController.signal.aborted) throw this.concluded("cancelled");
9684
+ if (this.abortController.signal.aborted) throw this.concludedOnAbort();
9637
9685
  if (apiErrorCode(err) === "execution_source_unavailable") return "refused";
9638
9686
  turnLog.warn(apiErrorLogFields(err), "dispatcher: failed to adopt the turn context");
9639
9687
  const reason = `execution_source_unavailable: ${describeError(err)}`;
@@ -10270,7 +10318,7 @@ var init_turn_execution = __esm({
10270
10318
  }
10271
10319
  if (abortController.signal.aborted) {
10272
10320
  o.okResult = false;
10273
- o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? "cancelled";
10321
+ o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? (this.terminatedFromOutside() ? RUNNER_TERMINATED : "cancelled");
10274
10322
  }
10275
10323
  const emptyResultReason = !this.skipState.skipped && classifyEmptyResult({
10276
10324
  ok: o.okResult,
@@ -10309,7 +10357,16 @@ var init_turn_execution = __esm({
10309
10357
  o.resultReason = "lease_lost";
10310
10358
  o.okResult = false;
10311
10359
  }
10312
- const userCancelled = abortController.signal.aborted && o.timeoutReason === null && !o.leaseLost;
10360
+ let terminated = this.terminatedFromOutside() && o.timeoutReason === null && !o.leaseLost;
10361
+ const cause = terminated ? await this.opts.inRunner?.terminationCause?.() : null;
10362
+ terminated &&= this.terminatedFromOutside();
10363
+ if (terminated) {
10364
+ o.resultReason = cause ?? RUNNER_TERMINATED;
10365
+ o.okResult = false;
10366
+ } else if (o.resultReason === RUNNER_TERMINATED) {
10367
+ o.resultReason = "cancelled";
10368
+ }
10369
+ const userCancelled = abortController.signal.aborted && o.timeoutReason === null && !o.leaseLost && !terminated;
10313
10370
  if (o.timeoutReason !== null) {
10314
10371
  o.resultReason = o.timeoutReason;
10315
10372
  o.okResult = false;
@@ -10807,6 +10864,9 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10807
10864
  let next = 0;
10808
10865
  let resolveFirst;
10809
10866
  let gotFirst = false;
10867
+ let stopRequested = false;
10868
+ let gone = false;
10869
+ const causeWaiters = /* @__PURE__ */ new Set();
10810
10870
  const first = new Promise((resolve2) => resolveFirst = resolve2);
10811
10871
  const lines2 = createInterface3({ input: stdin, crlfDelay: Infinity });
10812
10872
  lines2.on("line", (line) => {
@@ -10817,13 +10877,35 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10817
10877
  }
10818
10878
  const message2 = decodeLauncherMessage(line);
10819
10879
  if (message2?.cabaneLauncher === "granted") waiting.get(message2.id)?.();
10880
+ if (message2?.cabaneLauncher === "stop") stopRequested = true;
10881
+ if (message2?.cabaneLauncher === "cause") {
10882
+ for (const resolve2 of causeWaiters) resolve2(message2.cause);
10883
+ }
10820
10884
  });
10821
10885
  lines2.once("close", () => {
10886
+ gone = true;
10887
+ for (const resolve2 of causeWaiters) resolve2(null);
10822
10888
  if (!gotFirst) resolveFirst("");
10823
10889
  onLauncherGone();
10824
10890
  });
10825
10891
  return {
10826
10892
  first,
10893
+ get stopRequested() {
10894
+ return stopRequested;
10895
+ },
10896
+ terminationCause(timeoutMs = CAUSE_TIMEOUT_MS) {
10897
+ if (gone) return Promise.resolve(null);
10898
+ return new Promise((resolve2) => {
10899
+ const done = (cause) => {
10900
+ clearTimeout(timer);
10901
+ causeWaiters.delete(done);
10902
+ resolve2(cause);
10903
+ };
10904
+ const timer = setTimeout(() => done(null), timeoutMs);
10905
+ causeWaiters.add(done);
10906
+ emit({ cabaneRunner: "terminated" });
10907
+ });
10908
+ },
10827
10909
  acquire(key, signal) {
10828
10910
  const id = String(++next);
10829
10911
  return new Promise((resolve2, reject) => {
@@ -10850,10 +10932,12 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10850
10932
  }
10851
10933
  };
10852
10934
  }
10935
+ var CAUSE_TIMEOUT_MS;
10853
10936
  var init_launcher_channel = __esm({
10854
10937
  "src/runner/launcher-channel.ts"() {
10855
10938
  "use strict";
10856
10939
  init_protocol();
10940
+ CAUSE_TIMEOUT_MS = 2e3;
10857
10941
  }
10858
10942
  });
10859
10943
 
@@ -10950,6 +11034,8 @@ function runnerTurnOptions(input, log2, emit, extras = {}) {
10950
11034
  (sessionId) => emit({ cabaneRunner: "remote_session", serverUrl, sessionId })
10951
11035
  ),
10952
11036
  ...extras.onProcessGroup ? { onProcessGroup: extras.onProcessGroup } : {},
11037
+ ...extras.stopRequested ? { stopRequested: extras.stopRequested } : {},
11038
+ ...extras.terminationCause ? { terminationCause: extras.terminationCause } : {},
10953
11039
  codexExecutable: (path, containment) => {
10954
11040
  const prefix = containment.scopePrefix(process.env);
10955
11041
  return prefix ? shims.write([...prefix, path]) : path;
@@ -11007,16 +11093,16 @@ async function runTurn(env, deps) {
11007
11093
  const controllers = /* @__PURE__ */ new Set();
11008
11094
  const shims = createShimDir();
11009
11095
  let terminated = false;
11010
- const onTerm = () => {
11011
- terminated = true;
11012
- for (const controller of controllers) controller.abort();
11013
- };
11014
- deps.signals.on("SIGTERM", onTerm);
11015
11096
  const groups = [];
11016
11097
  const channel = openLauncherChannel(deps.stdin, emit, () => {
11017
11098
  if (deps.onLauncherGone) deps.onLauncherGone(groups);
11018
11099
  else void stopWithLauncher(groups);
11019
11100
  });
11101
+ const onTerm = () => {
11102
+ terminated = true;
11103
+ for (const controller of controllers) controller.abort(RUNNER_SIGTERM);
11104
+ };
11105
+ deps.signals.on("SIGTERM", onTerm);
11020
11106
  emit({ cabaneRunner: "ready" });
11021
11107
  try {
11022
11108
  const input = parseRunnerInput(env, await channel.first);
@@ -11026,7 +11112,7 @@ async function runTurn(env, deps) {
11026
11112
  const supervisor = {
11027
11113
  registerAbort: (_turnId, controller) => {
11028
11114
  controllers.add(controller);
11029
- if (terminated) controller.abort();
11115
+ if (terminated) controller.abort(RUNNER_SIGTERM);
11030
11116
  },
11031
11117
  releaseAbort: (_turnId, controller) => {
11032
11118
  controllers.delete(controller);
@@ -11045,6 +11131,8 @@ async function runTurn(env, deps) {
11045
11131
  ...deps.queryFn ? { queryFn: deps.queryFn } : {},
11046
11132
  shims,
11047
11133
  acquireLock: (key, signal) => channel.acquire(key, signal),
11134
+ stopRequested: () => channel.stopRequested,
11135
+ terminationCause: () => channel.terminationCause(),
11048
11136
  onProcessGroup: (pgid) => {
11049
11137
  groups.push(pgid);
11050
11138
  emit({ cabaneRunner: "process_group", pgid });
@@ -11073,6 +11161,7 @@ var init_run_turn = __esm({
11073
11161
  init_outbox();
11074
11162
  init_exec_shim();
11075
11163
  init_launcher_channel();
11164
+ init_turn_execution();
11076
11165
  init_opencode_lock();
11077
11166
  init_secrets();
11078
11167
  init_transcript_writer();
@@ -14993,8 +15082,6 @@ var CompanionSupervisor = class {
14993
15082
  const key = `${run.conversationId}|${run.agentId}`;
14994
15083
  if (wr.chains.has(key)) continue;
14995
15084
  this.recoveredTurns.add(run.turnId);
14996
- if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
14997
- continue;
14998
15085
  const payload = {
14999
15086
  type: "device:dispatch_requested",
15000
15087
  deviceId: this.deviceId,
@@ -15002,6 +15089,11 @@ var CompanionSupervisor = class {
15002
15089
  conversationId: run.conversationId,
15003
15090
  messageId: run.messageId
15004
15091
  };
15092
+ const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
15093
+ if (attempt > MAX_RESUME_ATTEMPTS) {
15094
+ await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
15095
+ continue;
15096
+ }
15005
15097
  const tail = this.runDispatch(
15006
15098
  wr,
15007
15099
  // Off `/runs`, not a frame, so no dispatch id to dedupe on;
@@ -15027,6 +15119,44 @@ var CompanionSupervisor = class {
15027
15119
  this.recovering = false;
15028
15120
  }
15029
15121
  }
15122
+ // An interrupted turn this device has tried to resume too often: it stops
15123
+ // trying, and the server is told the turn failed, so the conversation says it
15124
+ // couldn't resume instead of going quiet. The settle the launcher sends for a
15125
+ // turn it couldn't run: it ends the span under this id if one is still
15126
+ // running, disposes of the offer if none ever started, and is a no-op on a
15127
+ // turn that already ended. A settle that doesn't land leaves the turn to the
15128
+ // server's sweep, as before.
15129
+ async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
15130
+ this.log.error(
15131
+ {
15132
+ workspaceId,
15133
+ conversationId: payload.conversationId,
15134
+ agentId: payload.agentId,
15135
+ turnId,
15136
+ ...eventId ? { eventId } : {},
15137
+ attempt,
15138
+ cap: MAX_RESUME_ATTEMPTS
15139
+ },
15140
+ "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
15141
+ );
15142
+ try {
15143
+ await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
15144
+ activeRunStartedAt: null,
15145
+ turnId,
15146
+ outcome: "failed",
15147
+ errorReason: "resume_exhausted",
15148
+ readThrough: payload.messageId,
15149
+ // The delivery being given up on, so a newer one of the same message
15150
+ // is not failed in its name (the server checks under its lock).
15151
+ ...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
15152
+ });
15153
+ } catch (err) {
15154
+ this.log.warn(
15155
+ { ...apiErrorLogFields(err), turnId },
15156
+ "Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
15157
+ );
15158
+ }
15159
+ }
15030
15160
  // One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
15031
15161
  async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
15032
15162
  await this.initialHarnessReady;
@@ -15050,9 +15180,13 @@ var CompanionSupervisor = class {
15050
15180
  }
15051
15181
  const attempt = bumpResumeAttempt(workspaceId, dispatchId);
15052
15182
  if (attempt > MAX_RESUME_ATTEMPTS) {
15053
- this.log.error(
15054
- { workspaceId, eventId: dispatchId, attempt, cap: MAX_RESUME_ATTEMPTS },
15055
- "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
15183
+ await this.abandonResume(
15184
+ workspaceId,
15185
+ agent,
15186
+ payload,
15187
+ turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
15188
+ attempt,
15189
+ dispatchId
15056
15190
  );
15057
15191
  markCompleted(workspaceId, dispatchId);
15058
15192
  return;
package/dist/runtime.js CHANGED
@@ -7340,6 +7340,7 @@ import { promisify } from "util";
7340
7340
  var execFileAsync = promisify(execFile);
7341
7341
  var CONTAINMENT_STOP_GRACE_SEC = 5;
7342
7342
  var FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
7343
+ var OOM_READ_TIMEOUT_MS = 1500;
7343
7344
  function turnScopeUnit(turnId, attempt) {
7344
7345
  const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7345
7346
  return `cabane-turn-${safe}-${attempt}`;
@@ -7357,6 +7358,20 @@ async function reapTurnScopes(turnId) {
7357
7358
  return classifyStopError(message) === "absent" ? { failures: [] } : { failures: [message.slice(0, 200)] };
7358
7359
  }
7359
7360
  }
7361
+ async function turnScopeOomKilled(turnId) {
7362
+ if (process.platform !== "linux") return false;
7363
+ const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7364
+ try {
7365
+ const { stdout } = await execFileAsync(
7366
+ "systemctl",
7367
+ ["--user", "show", "--property=Result", "--value", `*-turn-${safe}-*.scope`],
7368
+ { timeout: OOM_READ_TIMEOUT_MS }
7369
+ );
7370
+ return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
7371
+ } catch {
7372
+ return false;
7373
+ }
7374
+ }
7360
7375
  function classifyStopError(message) {
7361
7376
  if (/failed to connect to bus|failed to get d-?bus connection|connection refused|refusing to operate|spawn systemctl|systemctl.*ENOENT|ENOENT.*systemctl/i.test(
7362
7377
  message
@@ -7697,6 +7712,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log) {
7697
7712
  async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7698
7713
  const startedAt = Date.now();
7699
7714
  const { turnId } = handleOpts;
7715
+ const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
7700
7716
  const log = opts.log.child({
7701
7717
  workspaceId: opts.workspaceId,
7702
7718
  conversationId: payload.conversationId,
@@ -7707,7 +7723,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7707
7723
  duties.registerAbort(turnId, controller);
7708
7724
  const stopRunner = new AbortController();
7709
7725
  let ready = false;
7726
+ let child;
7727
+ let stopAnnounced = false;
7728
+ const announceStop = () => {
7729
+ if (stopAnnounced || !child?.stdin.writable) return;
7730
+ stopAnnounced = true;
7731
+ child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
7732
+ };
7710
7733
  const forwardStop = () => {
7734
+ announceStop();
7711
7735
  if (ready) stopRunner.abort();
7712
7736
  };
7713
7737
  controller.signal.addEventListener("abort", forwardStop, { once: true });
@@ -7730,7 +7754,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7730
7754
  let thrown;
7731
7755
  let code;
7732
7756
  let handle;
7733
- let child;
7734
7757
  const groups = [];
7735
7758
  const remote = [];
7736
7759
  const lockOwner = (id) => `${turnId}#${id}`;
@@ -7778,6 +7801,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7778
7801
  stdin: runnerStdin(input)
7779
7802
  });
7780
7803
  child = running;
7804
+ if (controller.signal.aborted) announceStop();
7781
7805
  log.debug(
7782
7806
  { turnId, runnerPid: running.pid, provider: launch.provider.name },
7783
7807
  "runner: started"
@@ -7843,6 +7867,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7843
7867
  case "remote_session":
7844
7868
  remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
7845
7869
  return;
7870
+ case "terminated":
7871
+ void oomKilled(turnId).then((oom) => {
7872
+ if (running.stdin.writable) {
7873
+ running.stdin.write(
7874
+ encodeLauncherMessage({
7875
+ cabaneLauncher: "cause",
7876
+ cause: oom ? "out_of_memory" : null
7877
+ })
7878
+ );
7879
+ }
7880
+ });
7881
+ return;
7846
7882
  }
7847
7883
  });
7848
7884
  const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
@@ -7860,7 +7896,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7860
7896
  if (result) return result;
7861
7897
  const durationMs = Date.now() - startedAt;
7862
7898
  const stopped = controller.signal.aborted && thrown === void 0;
7863
- const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
7899
+ const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
7864
7900
  await child?.reap(groups);
7865
7901
  const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
7866
7902
  failures: [err instanceof Error ? err.message : String(err)]
@@ -10655,8 +10691,6 @@ var CompanionSupervisor = class {
10655
10691
  const key = `${run.conversationId}|${run.agentId}`;
10656
10692
  if (wr.chains.has(key)) continue;
10657
10693
  this.recoveredTurns.add(run.turnId);
10658
- if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
10659
- continue;
10660
10694
  const payload = {
10661
10695
  type: "device:dispatch_requested",
10662
10696
  deviceId: this.deviceId,
@@ -10664,6 +10698,11 @@ var CompanionSupervisor = class {
10664
10698
  conversationId: run.conversationId,
10665
10699
  messageId: run.messageId
10666
10700
  };
10701
+ const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
10702
+ if (attempt > MAX_RESUME_ATTEMPTS) {
10703
+ await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
10704
+ continue;
10705
+ }
10667
10706
  const tail = this.runDispatch(
10668
10707
  wr,
10669
10708
  // Off `/runs`, not a frame, so no dispatch id to dedupe on;
@@ -10689,6 +10728,44 @@ var CompanionSupervisor = class {
10689
10728
  this.recovering = false;
10690
10729
  }
10691
10730
  }
10731
+ // An interrupted turn this device has tried to resume too often: it stops
10732
+ // trying, and the server is told the turn failed, so the conversation says it
10733
+ // couldn't resume instead of going quiet. The settle the launcher sends for a
10734
+ // turn it couldn't run: it ends the span under this id if one is still
10735
+ // running, disposes of the offer if none ever started, and is a no-op on a
10736
+ // turn that already ended. A settle that doesn't land leaves the turn to the
10737
+ // server's sweep, as before.
10738
+ async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
10739
+ this.log.error(
10740
+ {
10741
+ workspaceId,
10742
+ conversationId: payload.conversationId,
10743
+ agentId: payload.agentId,
10744
+ turnId,
10745
+ ...eventId ? { eventId } : {},
10746
+ attempt,
10747
+ cap: MAX_RESUME_ATTEMPTS
10748
+ },
10749
+ "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
10750
+ );
10751
+ try {
10752
+ await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
10753
+ activeRunStartedAt: null,
10754
+ turnId,
10755
+ outcome: "failed",
10756
+ errorReason: "resume_exhausted",
10757
+ readThrough: payload.messageId,
10758
+ // The delivery being given up on, so a newer one of the same message
10759
+ // is not failed in its name (the server checks under its lock).
10760
+ ...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
10761
+ });
10762
+ } catch (err) {
10763
+ this.log.warn(
10764
+ { ...apiErrorLogFields(err), turnId },
10765
+ "Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
10766
+ );
10767
+ }
10768
+ }
10692
10769
  // One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
10693
10770
  async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
10694
10771
  await this.initialHarnessReady;
@@ -10712,9 +10789,13 @@ var CompanionSupervisor = class {
10712
10789
  }
10713
10790
  const attempt = bumpResumeAttempt(workspaceId, dispatchId);
10714
10791
  if (attempt > MAX_RESUME_ATTEMPTS) {
10715
- this.log.error(
10716
- { workspaceId, eventId: dispatchId, attempt, cap: MAX_RESUME_ATTEMPTS },
10717
- "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
10792
+ await this.abandonResume(
10793
+ workspaceId,
10794
+ agent,
10795
+ payload,
10796
+ turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
10797
+ attempt,
10798
+ dispatchId
10718
10799
  );
10719
10800
  markCompleted(workspaceId, dispatchId);
10720
10801
  return;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cabane/companion",
3
- "version": "0.6.194",
3
+ "version": "0.6.195",
4
4
  "type": "module",
5
5
  "description": "The Cabane Companion (headless): connect an agent on your machine to your Cabane workspace as a responder — drive work against your own codebase, files, and MCP servers without putting any of it in Cabane.",
6
6
  "license": "UNLICENSED",