@cabane/companion 0.6.194 → 0.6.196

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -7214,6 +7214,20 @@ async function reapTurnScopes(turnId) {
7214
7214
  return classifyStopError(message2) === "absent" ? { failures: [] } : { failures: [message2.slice(0, 200)] };
7215
7215
  }
7216
7216
  }
7217
+ async function turnScopeOomKilled(turnId) {
7218
+ if (process.platform !== "linux") return false;
7219
+ const safe2 = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7220
+ try {
7221
+ const { stdout } = await execFileAsync(
7222
+ "systemctl",
7223
+ ["--user", "show", "--property=Result", "--value", `*-turn-${safe2}-*.scope`],
7224
+ { timeout: OOM_READ_TIMEOUT_MS }
7225
+ );
7226
+ return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
7227
+ } catch {
7228
+ return false;
7229
+ }
7230
+ }
7217
7231
  function scopeCapabilityUsable(env, timeoutMs = CAPABILITY_PROBE_TIMEOUT_MS) {
7218
7232
  if (process.platform !== "linux") return false;
7219
7233
  try {
@@ -7462,7 +7476,7 @@ function createTurnContainment(turnId, deps = {}) {
7462
7476
  }
7463
7477
  };
7464
7478
  }
7465
- var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
7479
+ var execFileAsync, asContained, CONTAINMENT_STOP_GRACE_SEC, FALLBACK_GRACE_MS, CAPABILITY_PROBE_TIMEOUT_MS, OOM_READ_TIMEOUT_MS, sleep2, MANAGER_ENV_KEYS, managerEnvKey;
7466
7480
  var init_turn_containment = __esm({
7467
7481
  "src/turn-containment.ts"() {
7468
7482
  "use strict";
@@ -7471,6 +7485,7 @@ var init_turn_containment = __esm({
7471
7485
  CONTAINMENT_STOP_GRACE_SEC = 5;
7472
7486
  FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
7473
7487
  CAPABILITY_PROBE_TIMEOUT_MS = 4e3;
7488
+ OOM_READ_TIMEOUT_MS = 1500;
7474
7489
  sleep2 = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms));
7475
7490
  MANAGER_ENV_KEYS = ["DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR", "PATH"];
7476
7491
  managerEnvKey = (env) => MANAGER_ENV_KEYS.map((k) => `${k}=${env[k] ?? ""}`).join("\0");
@@ -7850,6 +7865,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log2) {
7850
7865
  async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7851
7866
  const startedAt = Date.now();
7852
7867
  const { turnId } = handleOpts;
7868
+ const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
7853
7869
  const log2 = opts.log.child({
7854
7870
  workspaceId: opts.workspaceId,
7855
7871
  conversationId: payload.conversationId,
@@ -7860,7 +7876,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7860
7876
  duties.registerAbort(turnId, controller);
7861
7877
  const stopRunner = new AbortController();
7862
7878
  let ready = false;
7879
+ let child;
7880
+ let stopAnnounced = false;
7881
+ const announceStop = () => {
7882
+ if (stopAnnounced || !child?.stdin.writable) return;
7883
+ stopAnnounced = true;
7884
+ child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
7885
+ };
7863
7886
  const forwardStop = () => {
7887
+ announceStop();
7864
7888
  if (ready) stopRunner.abort();
7865
7889
  };
7866
7890
  controller.signal.addEventListener("abort", forwardStop, { once: true });
@@ -7883,7 +7907,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7883
7907
  let thrown;
7884
7908
  let code;
7885
7909
  let handle;
7886
- let child;
7887
7910
  const groups = [];
7888
7911
  const remote = [];
7889
7912
  const lockOwner = (id) => `${turnId}#${id}`;
@@ -7931,6 +7954,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7931
7954
  stdin: runnerStdin(input)
7932
7955
  });
7933
7956
  child = running;
7957
+ if (controller.signal.aborted) announceStop();
7934
7958
  log2.debug(
7935
7959
  { turnId, runnerPid: running.pid, provider: launch.provider.name },
7936
7960
  "runner: started"
@@ -7996,6 +8020,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7996
8020
  case "remote_session":
7997
8021
  remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
7998
8022
  return;
8023
+ case "terminated":
8024
+ void oomKilled(turnId).then((oom) => {
8025
+ if (running.stdin.writable) {
8026
+ running.stdin.write(
8027
+ encodeLauncherMessage({
8028
+ cabaneLauncher: "cause",
8029
+ cause: oom ? "out_of_memory" : null
8030
+ })
8031
+ );
8032
+ }
8033
+ });
8034
+ return;
7999
8035
  }
8000
8036
  });
8001
8037
  const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
@@ -8013,7 +8049,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
8013
8049
  if (result) return result;
8014
8050
  const durationMs = Date.now() - startedAt;
8015
8051
  const stopped = controller.signal.aborted && thrown === void 0;
8016
- const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
8052
+ const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
8017
8053
  await child?.reap(groups);
8018
8054
  const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
8019
8055
  failures: [err instanceof Error ? err.message : String(err)]
@@ -9296,7 +9332,7 @@ function fingerprintSessionState(state) {
9296
9332
  }
9297
9333
  return createHash2("sha256").update(opaqueId).digest("hex").slice(0, 16);
9298
9334
  }
9299
- var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, TurnExecution;
9335
+ var PREPARING_TOOL_NAME, HARNESS_UNAVAILABLE_NOTICE, NO_TURN_TOKEN_REASON, UNEXPECTED_ROLE_REASON, DEFAULT_PREPARING_ROW_DELAY_MS, DEFAULT_AGENT_IDLE_TIMEOUT_MS, DEFAULT_AGENT_TOTAL_TIMEOUT_MS, DEFAULT_LEASE_RENEWAL_MS, TurnConcluded, RUNNER_SIGTERM, RUNNER_TERMINATED, TurnExecution;
9300
9336
  var init_turn_execution = __esm({
9301
9337
  "src/turn-execution.ts"() {
9302
9338
  "use strict";
@@ -9342,6 +9378,8 @@ var init_turn_execution = __esm({
9342
9378
  reason;
9343
9379
  errorReason;
9344
9380
  };
9381
+ RUNNER_SIGTERM = "runner_sigterm";
9382
+ RUNNER_TERMINATED = "runner_terminated";
9345
9383
  TurnExecution = class {
9346
9384
  constructor(opts, supervisor, payload, handleOpts = {}) {
9347
9385
  this.opts = opts;
@@ -9451,6 +9489,16 @@ var init_turn_execution = __esm({
9451
9489
  concluded(reason, errorReason) {
9452
9490
  return new TurnConcluded(reason, errorReason);
9453
9491
  }
9492
+ // A SIGTERM the launcher never announced as a Stop.
9493
+ terminatedFromOutside() {
9494
+ const { signal } = this.abortController;
9495
+ return signal.aborted && signal.reason === RUNNER_SIGTERM && !(this.opts.inRunner?.stopRequested?.() ?? false);
9496
+ }
9497
+ // An abort before the run: a Stop is silent, a termination nobody asked for
9498
+ // is reported.
9499
+ concludedOnAbort() {
9500
+ return this.terminatedFromOutside() ? this.concluded(RUNNER_TERMINATED, RUNNER_TERMINATED) : this.concluded("cancelled");
9501
+ }
9454
9502
  // The arc. Phases in order; every pre-loop exit throws `TurnConcluded` and
9455
9503
  // funnels through ONE conclusion (hand back the controller — idempotent and
9456
9504
  // identity-checked at the supervisor — then clear the active-run flag);
@@ -9571,7 +9619,7 @@ var init_turn_execution = __esm({
9571
9619
  } catch (err) {
9572
9620
  if (this.abortController.signal.aborted) {
9573
9621
  turnLog.debug("dispatcher: stopped while loading the turn context");
9574
- throw this.concluded("cancelled");
9622
+ throw this.concludedOnAbort();
9575
9623
  }
9576
9624
  const status2 = err instanceof ApiError ? err.status : 0;
9577
9625
  if (status2 === 404) {
@@ -9633,7 +9681,7 @@ var init_turn_execution = __esm({
9633
9681
  this.executionSourceId = adopted.executionSourceId;
9634
9682
  return "adopted";
9635
9683
  } catch (err) {
9636
- if (this.abortController.signal.aborted) throw this.concluded("cancelled");
9684
+ if (this.abortController.signal.aborted) throw this.concludedOnAbort();
9637
9685
  if (apiErrorCode(err) === "execution_source_unavailable") return "refused";
9638
9686
  turnLog.warn(apiErrorLogFields(err), "dispatcher: failed to adopt the turn context");
9639
9687
  const reason = `execution_source_unavailable: ${describeError(err)}`;
@@ -10270,7 +10318,7 @@ var init_turn_execution = __esm({
10270
10318
  }
10271
10319
  if (abortController.signal.aborted) {
10272
10320
  o.okResult = false;
10273
- o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? "cancelled";
10321
+ o.resultReason = o.leaseLost ? "lease_lost" : o.timeoutReason ?? (this.terminatedFromOutside() ? RUNNER_TERMINATED : "cancelled");
10274
10322
  }
10275
10323
  const emptyResultReason = !this.skipState.skipped && classifyEmptyResult({
10276
10324
  ok: o.okResult,
@@ -10309,7 +10357,16 @@ var init_turn_execution = __esm({
10309
10357
  o.resultReason = "lease_lost";
10310
10358
  o.okResult = false;
10311
10359
  }
10312
- const userCancelled = abortController.signal.aborted && o.timeoutReason === null && !o.leaseLost;
10360
+ let terminated = this.terminatedFromOutside() && o.timeoutReason === null && !o.leaseLost;
10361
+ const cause = terminated ? await this.opts.inRunner?.terminationCause?.() : null;
10362
+ terminated &&= this.terminatedFromOutside();
10363
+ if (terminated) {
10364
+ o.resultReason = cause ?? RUNNER_TERMINATED;
10365
+ o.okResult = false;
10366
+ } else if (o.resultReason === RUNNER_TERMINATED) {
10367
+ o.resultReason = "cancelled";
10368
+ }
10369
+ const userCancelled = abortController.signal.aborted && o.timeoutReason === null && !o.leaseLost && !terminated;
10313
10370
  if (o.timeoutReason !== null) {
10314
10371
  o.resultReason = o.timeoutReason;
10315
10372
  o.okResult = false;
@@ -10807,6 +10864,9 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10807
10864
  let next = 0;
10808
10865
  let resolveFirst;
10809
10866
  let gotFirst = false;
10867
+ let stopRequested = false;
10868
+ let gone = false;
10869
+ const causeWaiters = /* @__PURE__ */ new Set();
10810
10870
  const first = new Promise((resolve2) => resolveFirst = resolve2);
10811
10871
  const lines2 = createInterface3({ input: stdin, crlfDelay: Infinity });
10812
10872
  lines2.on("line", (line) => {
@@ -10817,13 +10877,35 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10817
10877
  }
10818
10878
  const message2 = decodeLauncherMessage(line);
10819
10879
  if (message2?.cabaneLauncher === "granted") waiting.get(message2.id)?.();
10880
+ if (message2?.cabaneLauncher === "stop") stopRequested = true;
10881
+ if (message2?.cabaneLauncher === "cause") {
10882
+ for (const resolve2 of causeWaiters) resolve2(message2.cause);
10883
+ }
10820
10884
  });
10821
10885
  lines2.once("close", () => {
10886
+ gone = true;
10887
+ for (const resolve2 of causeWaiters) resolve2(null);
10822
10888
  if (!gotFirst) resolveFirst("");
10823
10889
  onLauncherGone();
10824
10890
  });
10825
10891
  return {
10826
10892
  first,
10893
+ get stopRequested() {
10894
+ return stopRequested;
10895
+ },
10896
+ terminationCause(timeoutMs = CAUSE_TIMEOUT_MS) {
10897
+ if (gone) return Promise.resolve(null);
10898
+ return new Promise((resolve2) => {
10899
+ const done = (cause) => {
10900
+ clearTimeout(timer);
10901
+ causeWaiters.delete(done);
10902
+ resolve2(cause);
10903
+ };
10904
+ const timer = setTimeout(() => done(null), timeoutMs);
10905
+ causeWaiters.add(done);
10906
+ emit({ cabaneRunner: "terminated" });
10907
+ });
10908
+ },
10827
10909
  acquire(key, signal) {
10828
10910
  const id = String(++next);
10829
10911
  return new Promise((resolve2, reject) => {
@@ -10850,10 +10932,12 @@ function openLauncherChannel(stdin, emit, onLauncherGone) {
10850
10932
  }
10851
10933
  };
10852
10934
  }
10935
+ var CAUSE_TIMEOUT_MS;
10853
10936
  var init_launcher_channel = __esm({
10854
10937
  "src/runner/launcher-channel.ts"() {
10855
10938
  "use strict";
10856
10939
  init_protocol();
10940
+ CAUSE_TIMEOUT_MS = 2e3;
10857
10941
  }
10858
10942
  });
10859
10943
 
@@ -10950,6 +11034,8 @@ function runnerTurnOptions(input, log2, emit, extras = {}) {
10950
11034
  (sessionId) => emit({ cabaneRunner: "remote_session", serverUrl, sessionId })
10951
11035
  ),
10952
11036
  ...extras.onProcessGroup ? { onProcessGroup: extras.onProcessGroup } : {},
11037
+ ...extras.stopRequested ? { stopRequested: extras.stopRequested } : {},
11038
+ ...extras.terminationCause ? { terminationCause: extras.terminationCause } : {},
10953
11039
  codexExecutable: (path, containment) => {
10954
11040
  const prefix = containment.scopePrefix(process.env);
10955
11041
  return prefix ? shims.write([...prefix, path]) : path;
@@ -11007,16 +11093,16 @@ async function runTurn(env, deps) {
11007
11093
  const controllers = /* @__PURE__ */ new Set();
11008
11094
  const shims = createShimDir();
11009
11095
  let terminated = false;
11010
- const onTerm = () => {
11011
- terminated = true;
11012
- for (const controller of controllers) controller.abort();
11013
- };
11014
- deps.signals.on("SIGTERM", onTerm);
11015
11096
  const groups = [];
11016
11097
  const channel = openLauncherChannel(deps.stdin, emit, () => {
11017
11098
  if (deps.onLauncherGone) deps.onLauncherGone(groups);
11018
11099
  else void stopWithLauncher(groups);
11019
11100
  });
11101
+ const onTerm = () => {
11102
+ terminated = true;
11103
+ for (const controller of controllers) controller.abort(RUNNER_SIGTERM);
11104
+ };
11105
+ deps.signals.on("SIGTERM", onTerm);
11020
11106
  emit({ cabaneRunner: "ready" });
11021
11107
  try {
11022
11108
  const input = parseRunnerInput(env, await channel.first);
@@ -11026,7 +11112,7 @@ async function runTurn(env, deps) {
11026
11112
  const supervisor = {
11027
11113
  registerAbort: (_turnId, controller) => {
11028
11114
  controllers.add(controller);
11029
- if (terminated) controller.abort();
11115
+ if (terminated) controller.abort(RUNNER_SIGTERM);
11030
11116
  },
11031
11117
  releaseAbort: (_turnId, controller) => {
11032
11118
  controllers.delete(controller);
@@ -11045,6 +11131,8 @@ async function runTurn(env, deps) {
11045
11131
  ...deps.queryFn ? { queryFn: deps.queryFn } : {},
11046
11132
  shims,
11047
11133
  acquireLock: (key, signal) => channel.acquire(key, signal),
11134
+ stopRequested: () => channel.stopRequested,
11135
+ terminationCause: () => channel.terminationCause(),
11048
11136
  onProcessGroup: (pgid) => {
11049
11137
  groups.push(pgid);
11050
11138
  emit({ cabaneRunner: "process_group", pgid });
@@ -11073,6 +11161,7 @@ var init_run_turn = __esm({
11073
11161
  init_outbox();
11074
11162
  init_exec_shim();
11075
11163
  init_launcher_channel();
11164
+ init_turn_execution();
11076
11165
  init_opencode_lock();
11077
11166
  init_secrets();
11078
11167
  init_transcript_writer();
@@ -11287,7 +11376,6 @@ import { join as join7 } from "path";
11287
11376
  // src/device-state.ts
11288
11377
  var DEVICE_STATES = ["waiting", "online", "offline", "deactivated"];
11289
11378
  var DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS = 9e4;
11290
- var DEFAULT_DEVICE_RECONNECT_GRACE_MS = 9e4;
11291
11379
  var HARNESS_STATES = [
11292
11380
  "checking",
11293
11381
  "connected",
@@ -14993,8 +15081,6 @@ var CompanionSupervisor = class {
14993
15081
  const key = `${run.conversationId}|${run.agentId}`;
14994
15082
  if (wr.chains.has(key)) continue;
14995
15083
  this.recoveredTurns.add(run.turnId);
14996
- if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
14997
- continue;
14998
15084
  const payload = {
14999
15085
  type: "device:dispatch_requested",
15000
15086
  deviceId: this.deviceId,
@@ -15002,6 +15088,11 @@ var CompanionSupervisor = class {
15002
15088
  conversationId: run.conversationId,
15003
15089
  messageId: run.messageId
15004
15090
  };
15091
+ const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
15092
+ if (attempt > MAX_RESUME_ATTEMPTS) {
15093
+ await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
15094
+ continue;
15095
+ }
15005
15096
  const tail = this.runDispatch(
15006
15097
  wr,
15007
15098
  // Off `/runs`, not a frame, so no dispatch id to dedupe on;
@@ -15027,6 +15118,44 @@ var CompanionSupervisor = class {
15027
15118
  this.recovering = false;
15028
15119
  }
15029
15120
  }
15121
+ // An interrupted turn this device has tried to resume too often: it stops
15122
+ // trying, and the server is told the turn failed, so the conversation says it
15123
+ // couldn't resume instead of going quiet. The settle the launcher sends for a
15124
+ // turn it couldn't run: it ends the span under this id if one is still
15125
+ // running, disposes of the offer if none ever started, and is a no-op on a
15126
+ // turn that already ended. A settle that doesn't land leaves the turn to the
15127
+ // server's sweep, as before.
15128
+ async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
15129
+ this.log.error(
15130
+ {
15131
+ workspaceId,
15132
+ conversationId: payload.conversationId,
15133
+ agentId: payload.agentId,
15134
+ turnId,
15135
+ ...eventId ? { eventId } : {},
15136
+ attempt,
15137
+ cap: MAX_RESUME_ATTEMPTS
15138
+ },
15139
+ "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
15140
+ );
15141
+ try {
15142
+ await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
15143
+ activeRunStartedAt: null,
15144
+ turnId,
15145
+ outcome: "failed",
15146
+ errorReason: "resume_exhausted",
15147
+ readThrough: payload.messageId,
15148
+ // The delivery being given up on, so a newer one of the same message
15149
+ // is not failed in its name (the server checks under its lock).
15150
+ ...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
15151
+ });
15152
+ } catch (err) {
15153
+ this.log.warn(
15154
+ { ...apiErrorLogFields(err), turnId },
15155
+ "Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
15156
+ );
15157
+ }
15158
+ }
15030
15159
  // One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
15031
15160
  async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
15032
15161
  await this.initialHarnessReady;
@@ -15050,9 +15179,13 @@ var CompanionSupervisor = class {
15050
15179
  }
15051
15180
  const attempt = bumpResumeAttempt(workspaceId, dispatchId);
15052
15181
  if (attempt > MAX_RESUME_ATTEMPTS) {
15053
- this.log.error(
15054
- { workspaceId, eventId: dispatchId, attempt, cap: MAX_RESUME_ATTEMPTS },
15055
- "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
15182
+ await this.abandonResume(
15183
+ workspaceId,
15184
+ agent,
15185
+ payload,
15186
+ turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
15187
+ attempt,
15188
+ dispatchId
15056
15189
  );
15057
15190
  markCompleted(workspaceId, dispatchId);
15058
15191
  return;
@@ -16246,7 +16379,7 @@ function defaultSpawnDetached(args) {
16246
16379
  // src/terminal-copy.ts
16247
16380
  init_cabane_command();
16248
16381
  var UNPAIRED = "This device isn't paired yet. Get a setup line from a workspace's Settings \u2192 Devices and paste it here.";
16249
- var lease = DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS + DEFAULT_DEVICE_RECONNECT_GRACE_MS;
16382
+ var lease = DEFAULT_DEVICE_OFFLINE_THRESHOLD_MS;
16250
16383
  function age(at, now) {
16251
16384
  const seconds = Math.max(0, Math.floor((now - Date.parse(at)) / 1e3));
16252
16385
  return seconds < 60 ? `${seconds}s` : seconds < 3600 ? `${Math.floor(seconds / 60)}m` : `${Math.floor(seconds / 3600)}h`;
package/dist/runtime.js CHANGED
@@ -7340,6 +7340,7 @@ import { promisify } from "util";
7340
7340
  var execFileAsync = promisify(execFile);
7341
7341
  var CONTAINMENT_STOP_GRACE_SEC = 5;
7342
7342
  var FALLBACK_GRACE_MS = CONTAINMENT_STOP_GRACE_SEC * 1e3;
7343
+ var OOM_READ_TIMEOUT_MS = 1500;
7343
7344
  function turnScopeUnit(turnId, attempt) {
7344
7345
  const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7345
7346
  return `cabane-turn-${safe}-${attempt}`;
@@ -7357,6 +7358,20 @@ async function reapTurnScopes(turnId) {
7357
7358
  return classifyStopError(message) === "absent" ? { failures: [] } : { failures: [message.slice(0, 200)] };
7358
7359
  }
7359
7360
  }
7361
+ async function turnScopeOomKilled(turnId) {
7362
+ if (process.platform !== "linux") return false;
7363
+ const safe = turnId.replace(/[^A-Za-z0-9_.-]/g, "-");
7364
+ try {
7365
+ const { stdout } = await execFileAsync(
7366
+ "systemctl",
7367
+ ["--user", "show", "--property=Result", "--value", `*-turn-${safe}-*.scope`],
7368
+ { timeout: OOM_READ_TIMEOUT_MS }
7369
+ );
7370
+ return String(stdout).split("\n").some((line) => line.trim() === "oom-kill");
7371
+ } catch {
7372
+ return false;
7373
+ }
7374
+ }
7360
7375
  function classifyStopError(message) {
7361
7376
  if (/failed to connect to bus|failed to get d-?bus connection|connection refused|refusing to operate|spawn systemctl|systemctl.*ENOENT|ENOENT.*systemctl/i.test(
7362
7377
  message
@@ -7697,6 +7712,7 @@ async function settleUnstarted(opts, payload, turnId, errorReason, log) {
7697
7712
  async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7698
7713
  const startedAt = Date.now();
7699
7714
  const { turnId } = handleOpts;
7715
+ const oomKilled = (id) => (launch.oomKilled ?? turnScopeOomKilled)(id).catch(() => false);
7700
7716
  const log = opts.log.child({
7701
7717
  workspaceId: opts.workspaceId,
7702
7718
  conversationId: payload.conversationId,
@@ -7707,7 +7723,15 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7707
7723
  duties.registerAbort(turnId, controller);
7708
7724
  const stopRunner = new AbortController();
7709
7725
  let ready = false;
7726
+ let child;
7727
+ let stopAnnounced = false;
7728
+ const announceStop = () => {
7729
+ if (stopAnnounced || !child?.stdin.writable) return;
7730
+ stopAnnounced = true;
7731
+ child.stdin.write(encodeLauncherMessage({ cabaneLauncher: "stop" }));
7732
+ };
7710
7733
  const forwardStop = () => {
7734
+ announceStop();
7711
7735
  if (ready) stopRunner.abort();
7712
7736
  };
7713
7737
  controller.signal.addEventListener("abort", forwardStop, { once: true });
@@ -7730,7 +7754,6 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7730
7754
  let thrown;
7731
7755
  let code;
7732
7756
  let handle;
7733
- let child;
7734
7757
  const groups = [];
7735
7758
  const remote = [];
7736
7759
  const lockOwner = (id) => `${turnId}#${id}`;
@@ -7778,6 +7801,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7778
7801
  stdin: runnerStdin(input)
7779
7802
  });
7780
7803
  child = running;
7804
+ if (controller.signal.aborted) announceStop();
7781
7805
  log.debug(
7782
7806
  { turnId, runnerPid: running.pid, provider: launch.provider.name },
7783
7807
  "runner: started"
@@ -7843,6 +7867,18 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7843
7867
  case "remote_session":
7844
7868
  remote.push({ serverUrl: event.serverUrl, sessionId: event.sessionId });
7845
7869
  return;
7870
+ case "terminated":
7871
+ void oomKilled(turnId).then((oom) => {
7872
+ if (running.stdin.writable) {
7873
+ running.stdin.write(
7874
+ encodeLauncherMessage({
7875
+ cabaneLauncher: "cause",
7876
+ cause: oom ? "out_of_memory" : null
7877
+ })
7878
+ );
7879
+ }
7880
+ });
7881
+ return;
7846
7882
  }
7847
7883
  });
7848
7884
  const stderrDone = lines(running.stderr, (line) => relayRunnerLog(line, opts.log));
@@ -7860,7 +7896,7 @@ async function runTurnInRunner(opts, launch, duties, payload, handleOpts) {
7860
7896
  if (result) return result;
7861
7897
  const durationMs = Date.now() - startedAt;
7862
7898
  const stopped = controller.signal.aborted && thrown === void 0;
7863
- const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : `runner_exited: ${code}`;
7899
+ const reason = stopped ? "cancelled" : thrown !== void 0 ? `runner_failed: ${thrown}` : await oomKilled(turnId) ? "out_of_memory" : `runner_exited: ${code}`;
7864
7900
  await child?.reap(groups);
7865
7901
  const reaped = await (launch.reapTurn ?? reapTurnScopes)(turnId).catch((err) => ({
7866
7902
  failures: [err instanceof Error ? err.message : String(err)]
@@ -10655,8 +10691,6 @@ var CompanionSupervisor = class {
10655
10691
  const key = `${run.conversationId}|${run.agentId}`;
10656
10692
  if (wr.chains.has(key)) continue;
10657
10693
  this.recoveredTurns.add(run.turnId);
10658
- if (bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`) > MAX_RESUME_ATTEMPTS)
10659
- continue;
10660
10694
  const payload = {
10661
10695
  type: "device:dispatch_requested",
10662
10696
  deviceId: this.deviceId,
@@ -10664,6 +10698,11 @@ var CompanionSupervisor = class {
10664
10698
  conversationId: run.conversationId,
10665
10699
  messageId: run.messageId
10666
10700
  };
10701
+ const attempt = bumpResumeAttempt(run.workspaceId, `restart:${run.turnId}`);
10702
+ if (attempt > MAX_RESUME_ATTEMPTS) {
10703
+ await this.abandonResume(wr.workspaceId, agent, payload, run.turnId, attempt);
10704
+ continue;
10705
+ }
10667
10706
  const tail = this.runDispatch(
10668
10707
  wr,
10669
10708
  // Off `/runs`, not a frame, so no dispatch id to dedupe on;
@@ -10689,6 +10728,44 @@ var CompanionSupervisor = class {
10689
10728
  this.recovering = false;
10690
10729
  }
10691
10730
  }
10731
+ // An interrupted turn this device has tried to resume too often: it stops
10732
+ // trying, and the server is told the turn failed, so the conversation says it
10733
+ // couldn't resume instead of going quiet. The settle the launcher sends for a
10734
+ // turn it couldn't run: it ends the span under this id if one is still
10735
+ // running, disposes of the offer if none ever started, and is a no-op on a
10736
+ // turn that already ended. A settle that doesn't land leaves the turn to the
10737
+ // server's sweep, as before.
10738
+ async abandonResume(workspaceId, agent, payload, turnId, attempt, eventId) {
10739
+ this.log.error(
10740
+ {
10741
+ workspaceId,
10742
+ conversationId: payload.conversationId,
10743
+ agentId: payload.agentId,
10744
+ turnId,
10745
+ ...eventId ? { eventId } : {},
10746
+ attempt,
10747
+ cap: MAX_RESUME_ATTEMPTS
10748
+ },
10749
+ "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
10750
+ );
10751
+ try {
10752
+ await agent.api.setActiveRun(workspaceId, payload.conversationId, payload.agentId, {
10753
+ activeRunStartedAt: null,
10754
+ turnId,
10755
+ outcome: "failed",
10756
+ errorReason: "resume_exhausted",
10757
+ readThrough: payload.messageId,
10758
+ // The delivery being given up on, so a newer one of the same message
10759
+ // is not failed in its name (the server checks under its lock).
10760
+ ...payload.episode !== void 0 ? { dispatchMessageId: payload.messageId, dispatchEpisode: payload.episode } : {}
10761
+ });
10762
+ } catch (err) {
10763
+ this.log.warn(
10764
+ { ...apiErrorLogFields(err), turnId },
10765
+ "Couldn't tell Cabane the interrupted turn won't resume. Check the conversation."
10766
+ );
10767
+ }
10768
+ }
10692
10769
  // One turn: the skip-vs-resume dedupe, the dispatch, then the console summary.
10693
10770
  async runDispatch(wr, dispatchId, payload, agent, recoveredTurnId) {
10694
10771
  await this.initialHarnessReady;
@@ -10712,9 +10789,13 @@ var CompanionSupervisor = class {
10712
10789
  }
10713
10790
  const attempt = bumpResumeAttempt(workspaceId, dispatchId);
10714
10791
  if (attempt > MAX_RESUME_ATTEMPTS) {
10715
- this.log.error(
10716
- { workspaceId, eventId: dispatchId, attempt, cap: MAX_RESUME_ATTEMPTS },
10717
- "An interrupted turn couldn't finish after several attempts. Reply in Cabane to try again."
10792
+ await this.abandonResume(
10793
+ workspaceId,
10794
+ agent,
10795
+ payload,
10796
+ turnIdForEvent(workspaceId, dispatchId) ?? randomUUID5(),
10797
+ attempt,
10798
+ dispatchId
10718
10799
  );
10719
10800
  markCompleted(workspaceId, dispatchId);
10720
10801
  return;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cabane/companion",
3
- "version": "0.6.194",
3
+ "version": "0.6.196",
4
4
  "type": "module",
5
5
  "description": "The Cabane Companion (headless): connect an agent on your machine to your Cabane workspace as a responder — drive work against your own codebase, files, and MCP servers without putting any of it in Cabane.",
6
6
  "license": "UNLICENSED",