@moxt-ai/cli 0.10.0-beta.7 → 0.10.0-beta.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -8185,11 +8185,11 @@ var init_app_server_client = __esm({
8185
8185
  });
8186
8186
 
8187
8187
  // ../machine-connector/dist/runtime/local-codex-runtime/durable-attempt.js
8188
- function assertCompatibleClaim(persisted, connectionId, claim) {
8188
+ function assertCompatibleClaim(persisted, connectionId, claim, allowRestartFenceRotation) {
8189
8189
  if (persisted.connectionId !== connectionId || persisted.claim.attemptId !== claim.attemptId || persisted.claim.attemptGeneration !== claim.attemptGeneration || persisted.claim.conversationId !== claim.conversationId || persisted.claim.turnId !== claim.turnId || persisted.claim.expectedNativeStoreIdentity !== claim.expectedNativeStoreIdentity) {
8190
8190
  throw new Error(`Local Codex Runtime attempt ${claim.attemptId} conflicts with its durable spool`);
8191
8191
  }
8192
- if (persisted.phase !== LocalCodexAttemptPhase.Claimed && (persisted.claim.leaseToken !== claim.leaseToken || persisted.claim.runToken !== claim.runToken)) {
8192
+ if (persisted.phase !== LocalCodexAttemptPhase.Claimed && !allowRestartFenceRotation && (persisted.claim.leaseToken !== claim.leaseToken || persisted.claim.runToken !== claim.runToken)) {
8193
8193
  throw new Error(`Running Local Codex Runtime attempt ${claim.attemptId} changed its fence`);
8194
8194
  }
8195
8195
  }
@@ -8217,16 +8217,19 @@ var init_durable_attempt = __esm({
8217
8217
  DurableLocalCodexAttempt = class _DurableLocalCodexAttempt {
8218
8218
  store;
8219
8219
  state;
8220
+ recoveryRequired;
8220
8221
  mutationQueue = Promise.resolve();
8221
- constructor(store, state) {
8222
+ constructor(store, state, recoveryRequired = false) {
8222
8223
  this.store = store;
8223
8224
  this.state = state;
8225
+ this.recoveryRequired = recoveryRequired;
8224
8226
  }
8225
8227
  static async create(store, connectionId, connectorInstanceId, claim) {
8226
8228
  const persisted = (await store.readLocalCodexAttempt(claim.attemptId)).value;
8227
8229
  if (persisted !== void 0) {
8228
- assertCompatibleClaim(persisted, connectionId, claim);
8229
- if (persisted.phase === LocalCodexAttemptPhase.Claimed) {
8230
+ const recoveryRequired = persisted.phase === LocalCodexAttemptPhase.TerminalPendingUpload || persisted.phase === LocalCodexAttemptPhase.Running && persisted.connectorInstanceId !== connectorInstanceId;
8231
+ assertCompatibleClaim(persisted, connectionId, claim, recoveryRequired);
8232
+ if (persisted.phase === LocalCodexAttemptPhase.Claimed || recoveryRequired) {
8230
8233
  persisted.claim = claim;
8231
8234
  persisted.connectorInstanceId = connectorInstanceId;
8232
8235
  persisted.acknowledgedSequence = claim.lastAcknowledgedEventSequence;
@@ -8235,7 +8238,7 @@ var init_durable_attempt = __esm({
8235
8238
  persisted.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
8236
8239
  await store.writeLocalCodexAttempt(persisted);
8237
8240
  }
8238
- return new _DurableLocalCodexAttempt(store, persisted);
8241
+ return new _DurableLocalCodexAttempt(store, persisted, recoveryRequired);
8239
8242
  }
8240
8243
  const state = {
8241
8244
  acknowledgedSequence: claim.lastAcknowledgedEventSequence,
@@ -8258,6 +8261,9 @@ var init_durable_attempt = __esm({
8258
8261
  snapshot() {
8259
8262
  return structuredClone(this.state);
8260
8263
  }
8264
+ requiresRecovery() {
8265
+ return this.recoveryRequired;
8266
+ }
8261
8267
  async markRunning(nativeThreadId, nativeTurnId) {
8262
8268
  await this.mutate(async () => {
8263
8269
  this.state.nativeThreadId = nativeThreadId;
@@ -8728,9 +8734,11 @@ function createActiveAttempt(claim, durable, workingDirectory) {
8728
8734
  eventUploadQueue: Promise.resolve(),
8729
8735
  fatal,
8730
8736
  finished: new AbortController(),
8737
+ ignoreProcessLoss: false,
8731
8738
  interruptedCleanupStarted: false,
8732
8739
  leaseDeadlineMilliseconds: parseLeaseDeadline(claim.leaseExpiresAt),
8733
8740
  nativeThreadId: durable.snapshot().nativeThreadId ?? claim.nativeThreadId,
8741
+ processQuarantined: false,
8734
8742
  spoolQueue: Promise.resolve(),
8735
8743
  subagentThreadIds: /* @__PURE__ */ new Set(),
8736
8744
  terminal,
@@ -8820,6 +8828,24 @@ function appServerMessageThreadId(message) {
8820
8828
  }
8821
8829
  return optionalString2(safeRecord(params.thread).id);
8822
8830
  }
8831
+ function appServerMessageTurnId(message) {
8832
+ const params = safeRecord(message.params);
8833
+ const direct = optionalString2(params.turnId).value;
8834
+ if (direct !== void 0) {
8835
+ return { value: direct };
8836
+ }
8837
+ return optionalString2(safeRecord(params.turn).id);
8838
+ }
8839
+ function activeOwnsThread(active, threadId) {
8840
+ return active.nativeThreadId === threadId || active.subagentThreadIds.has(threadId) || active.initialThreadName?.generation?.threadId === threadId;
8841
+ }
8842
+ function singleActiveAttempt(attempts) {
8843
+ if (attempts.length !== 1) {
8844
+ return {};
8845
+ }
8846
+ const value = attempts[0];
8847
+ return value === void 0 ? {} : { value };
8848
+ }
8823
8849
  function normalizeThreadTitle(value) {
8824
8850
  const title = Array.from(value.trim().replace(/\s+/g, " ")).slice(0, maximumThreadTitleCharacters).join("");
8825
8851
  return title === "" ? {} : { value: title };
@@ -9120,6 +9146,23 @@ async function waitForAbort(signal, milliseconds) {
9120
9146
  signal.addEventListener("abort", complete, { once: true });
9121
9147
  });
9122
9148
  }
9149
+ function createReclaimableClaimWindow() {
9150
+ let close = () => void 0;
9151
+ const completion = new Promise((resolvePromise) => {
9152
+ close = resolvePromise;
9153
+ });
9154
+ return { close, completion };
9155
+ }
9156
+ function rejectWhenAborted(signal) {
9157
+ return new Promise((_resolve, reject) => {
9158
+ const rejectStopped = () => reject(new WorkerStoppedError());
9159
+ if (signal.aborted) {
9160
+ rejectStopped();
9161
+ return;
9162
+ }
9163
+ signal.addEventListener("abort", rejectStopped, { once: true });
9164
+ });
9165
+ }
9123
9166
  function parseLeaseDeadline(value) {
9124
9167
  const parsed = Date.parse(value);
9125
9168
  if (!Number.isFinite(parsed)) {
@@ -9127,7 +9170,7 @@ function parseLeaseDeadline(value) {
9127
9170
  }
9128
9171
  return parsed;
9129
9172
  }
9130
- var import_node_child_process4, import_promises13, import_node_os7, import_node_path11, heartbeatIntervalMilliseconds, claimRetryMilliseconds, claimWaitSeconds, discoveryWaitSeconds, discoveryRetryMilliseconds, maximumDiscoveryDirectories, maximumDiscoveryModels, maximumModelListPages, eventUploadCoalesceMilliseconds, eventUploadRetryMilliseconds, maximumEventUploadCount, preferredEventUploadRequestBytes, leaseRenewIntervalMilliseconds, leaseRenewRetryMilliseconds, interactionTimeoutMilliseconds, executableProbeTimeoutMilliseconds, recoveryRetryMilliseconds, initialThreadNameRetryMilliseconds, maximumThreadNameRetryMilliseconds, maximumInitialThreadNameAttempts, maximumThreadTitleCharacters, maximumThreadTitleSourceCharacters, threadTitleGenerationTimeoutMilliseconds, interruptedExecutionCleanupTimeoutMilliseconds, threadTitleGenerationInstructions, threadTitleGenerationOutputSchema, ThreadUnavailableError, LeaseLostError, SpoolPersistenceError, UnknownTurnStartResultError, LocalCodexRuntimeConnectionWorker;
9173
+ var import_node_child_process4, import_promises13, import_node_os7, import_node_path11, heartbeatIntervalMilliseconds, claimRetryMilliseconds, claimWaitSeconds, discoveryWaitSeconds, discoveryRetryMilliseconds, maximumDiscoveryDirectories, maximumDiscoveryModels, maximumModelListPages, eventUploadCoalesceMilliseconds, eventUploadRetryMilliseconds, maximumEventUploadCount, preferredEventUploadRequestBytes, leaseRenewIntervalMilliseconds, leaseRenewRetryMilliseconds, interactionTimeoutMilliseconds, executableProbeTimeoutMilliseconds, recoveryRetryMilliseconds, initialThreadNameRetryMilliseconds, maximumThreadNameRetryMilliseconds, maximumInitialThreadNameAttempts, maximumThreadTitleCharacters, maximumThreadTitleSourceCharacters, threadTitleGenerationTimeoutMilliseconds, interruptedExecutionCleanupTimeoutMilliseconds, threadTitleGenerationInstructions, threadTitleGenerationOutputSchema, ThreadUnavailableError, LeaseLostError, SpoolPersistenceError, UnknownTurnStartResultError, WorkerStoppedError, LocalCodexRuntimeConnectionWorker;
9131
9174
  var init_runtime_worker = __esm({
9132
9175
  "../machine-connector/dist/runtime/local-codex-runtime/runtime-worker.js"() {
9133
9176
  "use strict";
@@ -9203,6 +9246,12 @@ var init_runtime_worker = __esm({
9203
9246
  this.name = "UnknownTurnStartResultError";
9204
9247
  }
9205
9248
  };
9249
+ WorkerStoppedError = class extends Error {
9250
+ constructor() {
9251
+ super("Local Codex Runtime worker stopped");
9252
+ this.name = "WorkerStoppedError";
9253
+ }
9254
+ };
9206
9255
  LocalCodexRuntimeConnectionWorker = class {
9207
9256
  connection;
9208
9257
  relay;
@@ -9211,9 +9260,10 @@ var init_runtime_worker = __esm({
9211
9260
  environment;
9212
9261
  connectorInstanceId;
9213
9262
  logger;
9214
- active = {};
9263
+ activeAttempts = /* @__PURE__ */ new Map();
9215
9264
  configured = false;
9216
9265
  enabled = false;
9266
+ executionTasks = /* @__PURE__ */ new Map();
9217
9267
  lastCapabilityStartError;
9218
9268
  lifetimeAbort = new AbortController();
9219
9269
  registered = true;
@@ -9242,13 +9292,55 @@ var init_runtime_worker = __esm({
9242
9292
  }
9243
9293
  revoke() {
9244
9294
  this.lifetimeAbort.abort();
9245
- this.active.value?.finished.abort();
9295
+ for (const active of this.activeAttempts.values()) {
9296
+ active.finished.abort();
9297
+ }
9246
9298
  }
9247
9299
  status() {
9248
9300
  return this.processPool.status();
9249
9301
  }
9302
+ ownsMessage(message) {
9303
+ return this.activeAttemptForMessage(message).value !== void 0;
9304
+ }
9305
+ activeAttemptForMessage(message) {
9306
+ const threadId = appServerMessageThreadId(message).value;
9307
+ const turnId = appServerMessageTurnId(message).value;
9308
+ if (threadId === void 0 && turnId === void 0) {
9309
+ return {};
9310
+ }
9311
+ if (turnId !== void 0) {
9312
+ const exactTurnMatches = [...this.activeAttempts.values()].filter((active) => {
9313
+ if (active.finished.signal.aborted) {
9314
+ return false;
9315
+ }
9316
+ const generation = active.initialThreadName?.generation;
9317
+ const ownsTurn = active.nativeTurnId === turnId || generation?.turnId === turnId;
9318
+ return ownsTurn && (threadId === void 0 || activeOwnsThread(active, threadId));
9319
+ });
9320
+ if (exactTurnMatches.length > 0) {
9321
+ return singleActiveAttempt(exactTurnMatches);
9322
+ }
9323
+ }
9324
+ if (threadId === void 0) {
9325
+ return {};
9326
+ }
9327
+ const threadMatches = [...this.activeAttempts.values()].filter((active) => {
9328
+ if (active.finished.signal.aborted) {
9329
+ return false;
9330
+ }
9331
+ const generation = active.initialThreadName?.generation;
9332
+ const ownsVisibleThread = active.nativeThreadId === threadId;
9333
+ const ownsSubagentThread = active.subagentThreadIds.has(threadId);
9334
+ const ownsTitleThread = generation?.threadId === threadId;
9335
+ if (turnId === void 0) {
9336
+ return ownsVisibleThread || ownsSubagentThread || ownsTitleThread;
9337
+ }
9338
+ return ownsSubagentThread || ownsVisibleThread && active.nativeTurnId === void 0 || ownsTitleThread && generation?.turnId === void 0;
9339
+ });
9340
+ return singleActiveAttempt(threadMatches);
9341
+ }
9250
9342
  async onNotification(message) {
9251
- const active = this.active.value;
9343
+ const active = this.activeAttemptForMessage(message).value;
9252
9344
  if (active === void 0 || active.finished.signal.aborted) {
9253
9345
  return;
9254
9346
  }
@@ -9290,7 +9382,13 @@ var init_runtime_worker = __esm({
9290
9382
  initialThreadName.terminalObserved = true;
9291
9383
  }
9292
9384
  if (normalized.terminalStatus === LocalCodexTurnStatusWire.Interrupted) {
9293
- await this.stopInterruptedExecution(active, normalized);
9385
+ if (active.interruptedCleanupTask === void 0) {
9386
+ const task = this.stopInterruptedExecution(active, normalized).catch((error) => {
9387
+ this.signalFatal(active, error instanceof Error ? error : new Error(String(error)));
9388
+ });
9389
+ active.interruptedCleanupTask = task;
9390
+ void task;
9391
+ }
9294
9392
  return;
9295
9393
  }
9296
9394
  if (normalized.terminalStatus !== void 0 && initialThreadName !== void 0 && !initialThreadName.completed) {
@@ -9317,46 +9415,60 @@ var init_runtime_worker = __esm({
9317
9415
  void this.enqueueEventFlush(active).catch(() => void 0);
9318
9416
  }
9319
9417
  async stopInterruptedExecution(active, normalized) {
9418
+ if (active.interruptedCleanupStarted) {
9419
+ return;
9420
+ }
9320
9421
  active.interruptedCleanupStarted = true;
9422
+ active.ignoreProcessLoss = true;
9321
9423
  active.initialThreadName?.generation?.resolve();
9322
9424
  const client = this.processPool.currentClient().value;
9425
+ let cleanupFailed = false;
9323
9426
  if (client !== void 0 && active.nativeThreadId !== void 0) {
9324
9427
  await Promise.all([active.nativeThreadId, ...active.subagentThreadIds].map(async (threadId) => {
9325
9428
  try {
9326
9429
  await client.requestWithin("thread/backgroundTerminals/clean", { threadId }, interruptedExecutionCleanupTimeoutMilliseconds);
9327
9430
  } catch (error) {
9328
- await this.logger.error(`Failed to clean background terminals for interrupted Local Codex thread ${threadId}; stopping its App Server process tree`, error);
9431
+ cleanupFailed = true;
9432
+ await this.logger.error(`Failed to clean background terminals for interrupted Local Codex thread ${threadId}; quarantining the shared App Server`, error);
9329
9433
  }
9330
9434
  }));
9331
9435
  }
9332
- try {
9333
- await this.processPool.quarantineCurrentProcess(active.processTreeCapture);
9334
- } catch (error) {
9335
- const failure = error instanceof Error ? error : new AppServerProcessLostError(String(error));
9336
- await this.enqueueSpool(active, async () => {
9337
- const terminal = await active.durable.appendEvent("turn_failed", {
9338
- message: failure.message,
9339
- status: "runtime_lost"
9340
- }, {
9341
- nativeThreadId: active.nativeThreadId,
9342
- nativeTurnId: active.nativeTurnId
9436
+ const quarantineProcess = cleanupFailed || !this.processPool.hasOtherActiveExecution();
9437
+ if (quarantineProcess) {
9438
+ active.processQuarantined = true;
9439
+ try {
9440
+ if (client === void 0) {
9441
+ await this.processPool.quarantineCurrentProcess(active.processTreeCapture);
9442
+ } else {
9443
+ await this.processPool.quarantineProcess(client, active.processTreeCapture);
9444
+ }
9445
+ } catch (error) {
9446
+ const failure = error instanceof Error ? error : new AppServerProcessLostError(String(error));
9447
+ await this.enqueueSpool(active, async () => {
9448
+ const terminal = await active.durable.appendEvent("turn_failed", {
9449
+ message: failure.message,
9450
+ status: "runtime_lost"
9451
+ }, {
9452
+ nativeThreadId: active.nativeThreadId,
9453
+ nativeTurnId: active.nativeTurnId
9454
+ });
9455
+ const result = {
9456
+ errorCode: "runtime_lost",
9457
+ errorMessage: failure.message,
9458
+ finalSequence: terminal.sequence,
9459
+ status: LocalCodexTurnStatusWire.RuntimeLost
9460
+ };
9461
+ await active.durable.markTerminal(result);
9462
+ this.settleTerminal(active, result);
9343
9463
  });
9344
- const result = {
9345
- errorCode: "runtime_lost",
9346
- errorMessage: failure.message,
9347
- finalSequence: terminal.sequence,
9348
- status: LocalCodexTurnStatusWire.RuntimeLost
9349
- };
9350
- await active.durable.markTerminal(result);
9351
- this.settleTerminal(active, result);
9352
- });
9353
- return;
9464
+ return;
9465
+ }
9354
9466
  }
9355
9467
  this.completeInitialThreadName(active);
9356
9468
  this.persistNormalizedNotification(active, normalized);
9357
9469
  }
9358
9470
  async onServerRequest(message) {
9359
- const active = this.active.value;
9471
+ const active = this.activeAttemptForMessage(message).value;
9360
9472
  const client = this.processPool.currentClient().value;
9361
9473
  const requestId = rpcIdentifier2(message.id).value;
9362
9474
  if (active === void 0 || client === void 0 || requestId === void 0 || message.method === void 0) {
@@ -9398,46 +9510,50 @@ var init_runtime_worker = __esm({
9398
9510
  await this.enqueueEventFlush(active);
9399
9511
  }
9400
9512
  async onProcessExit(error) {
9401
- const active = this.active.value;
9402
- if (active === void 0 || !active.acceptedByAppServer) {
9403
- return;
9404
- }
9405
- active.initialThreadName?.generation?.resolve();
9406
- await this.enqueueSpool(active, async () => {
9407
- await active.durable.appendEvent("runtime_error", { message: error.message }, {
9408
- nativeThreadId: active.nativeThreadId,
9409
- nativeTurnId: active.nativeTurnId
9410
- });
9411
- const terminal = await active.durable.appendEvent("turn_failed", { status: "runtime_lost" }, {
9412
- nativeThreadId: active.nativeThreadId,
9413
- nativeTurnId: active.nativeTurnId
9513
+ await Promise.all([...this.activeAttempts.values()].filter((active) => active.acceptedByAppServer && !active.finished.signal.aborted && !active.ignoreProcessLoss && active.terminalValue === void 0).map(async (active) => {
9514
+ active.initialThreadName?.generation?.resolve();
9515
+ await this.enqueueSpool(active, async () => {
9516
+ await active.durable.appendEvent("runtime_error", { message: error.message }, {
9517
+ nativeThreadId: active.nativeThreadId,
9518
+ nativeTurnId: active.nativeTurnId
9519
+ });
9520
+ const terminal = await active.durable.appendEvent("turn_failed", { status: "runtime_lost" }, {
9521
+ nativeThreadId: active.nativeThreadId,
9522
+ nativeTurnId: active.nativeTurnId
9523
+ });
9524
+ const result = {
9525
+ errorCode: "runtime_lost",
9526
+ errorMessage: error.message,
9527
+ finalSequence: terminal.sequence,
9528
+ status: LocalCodexTurnStatusWire.RuntimeLost
9529
+ };
9530
+ await active.durable.markTerminal(result);
9531
+ this.settleTerminal(active, result);
9414
9532
  });
9415
- const result = {
9416
- errorCode: "runtime_lost",
9417
- errorMessage: error.message,
9418
- finalSequence: terminal.sequence,
9419
- status: LocalCodexTurnStatusWire.RuntimeLost
9420
- };
9421
- await active.durable.markTerminal(result);
9422
- this.settleTerminal(active, result);
9423
- });
9533
+ }));
9424
9534
  }
9425
9535
  async runLoops() {
9426
9536
  try {
9427
9537
  await Promise.all([
9428
9538
  this.runHeartbeatLoop(),
9429
- this.runClaimLoop(),
9430
9539
  this.runDiscoveryLoop(),
9431
- this.runRecoveryLoop()
9540
+ this.runAttemptLoops()
9432
9541
  ]);
9433
9542
  } finally {
9434
- this.active.value?.finished.abort();
9543
+ for (const active of this.activeAttempts.values()) {
9544
+ active.finished.abort();
9545
+ }
9546
+ await Promise.all(this.executionTasks.values());
9435
9547
  this.unregisterRoute();
9436
9548
  }
9437
9549
  }
9550
+ async runAttemptLoops() {
9551
+ await this.scanPersistedAttempts();
9552
+ await Promise.all([this.runClaimLoop(), this.runRecoveryLoop()]);
9553
+ }
9438
9554
  async runHeartbeatLoop() {
9439
9555
  while (!this.lifetimeAbort.signal.aborted && !this.relay.isRevoked()) {
9440
- if (this.active.value?.interruptedCleanupStarted) {
9556
+ if ([...this.activeAttempts.values()].some((active) => active.interruptedCleanupStarted)) {
9441
9557
  await waitForAbort(this.lifetimeAbort.signal, claimRetryMilliseconds);
9442
9558
  continue;
9443
9559
  }
@@ -9459,10 +9575,12 @@ var init_runtime_worker = __esm({
9459
9575
  await waitForAbort(this.lifetimeAbort.signal, claimRetryMilliseconds);
9460
9576
  continue;
9461
9577
  }
9462
- if (!this.processPool.tryAcquire(this.connection.connectionId)) {
9578
+ const slot = this.processPool.tryAcquireExecutionSlot(this.connection.connectionId).value;
9579
+ if (slot === void 0) {
9463
9580
  await waitForAbort(this.lifetimeAbort.signal, claimRetryMilliseconds);
9464
9581
  continue;
9465
9582
  }
9583
+ let executionScheduled = false;
9466
9584
  try {
9467
9585
  const capability = this.processPool.status();
9468
9586
  const response = await this.relay.claimCodexRuntime({
@@ -9473,21 +9591,37 @@ var init_runtime_worker = __esm({
9473
9591
  waitSeconds: claimWaitSeconds
9474
9592
  }, this.lifetimeAbort.signal);
9475
9593
  if (response.attempt === void 0) {
9476
- this.processPool.release(this.connection.connectionId);
9477
9594
  continue;
9478
9595
  }
9479
- await this.executeAttempt(response.attempt);
9596
+ const reclaimableClaimWindow = this.scheduleExecution(slot, response.attempt);
9597
+ executionScheduled = true;
9598
+ await reclaimableClaimWindow;
9480
9599
  } catch (error) {
9481
9600
  if (!this.lifetimeAbort.signal.aborted) {
9482
9601
  await this.logger.error(`Local Codex Runtime claim loop failed for connection ${this.connection.connectionId}`, error);
9483
9602
  await waitForAbort(this.lifetimeAbort.signal, eventUploadRetryMilliseconds);
9484
9603
  }
9485
9604
  } finally {
9486
- this.active = {};
9487
- this.processPool.release(this.connection.connectionId);
9605
+ if (!executionScheduled) {
9606
+ this.processPool.releaseExecutionSlot(slot);
9607
+ }
9488
9608
  }
9489
9609
  }
9490
9610
  }
9611
+ scheduleExecution(slot, claim) {
9612
+ if (this.executionTasks.has(claim.attemptId)) {
9613
+ throw new Error(`Local Codex attempt ${claim.attemptId} is already executing`);
9614
+ }
9615
+ this.processPool.markExecutionStarted(slot);
9616
+ const reclaimableClaimWindow = createReclaimableClaimWindow();
9617
+ const task = this.executeAttempt(claim, reclaimableClaimWindow).catch(async (error) => await this.logger.error(`Local Codex attempt ${claim.attemptId} execution crashed`, error).catch(() => void 0)).finally(() => {
9618
+ reclaimableClaimWindow.close();
9619
+ this.executionTasks.delete(claim.attemptId);
9620
+ this.processPool.releaseExecutionSlot(slot);
9621
+ });
9622
+ this.executionTasks.set(claim.attemptId, task);
9623
+ return reclaimableClaimWindow.completion;
9624
+ }
9491
9625
  async runDiscoveryLoop() {
9492
9626
  while (!this.lifetimeAbort.signal.aborted && !this.relay.isRevoked()) {
9493
9627
  try {
@@ -9587,10 +9721,17 @@ var init_runtime_worker = __esm({
9587
9721
  truncated: eligibleDirectories.length > maximumDiscoveryDirectories
9588
9722
  };
9589
9723
  }
9590
- async executeAttempt(claim) {
9724
+ async executeAttempt(claim, reclaimableClaimWindow) {
9591
9725
  const durable = await DurableLocalCodexAttempt.create(this.store, this.connection.connectionId, this.connectorInstanceId, claim);
9726
+ if (durable.requiresRecovery()) {
9727
+ await this.recoverPersistedAttempt(durable.snapshot());
9728
+ return;
9729
+ }
9592
9730
  const active = createActiveAttempt(claim, durable, this.connection.rootPath);
9593
- this.active = { value: active };
9731
+ if (this.activeAttempts.has(claim.attemptId)) {
9732
+ throw new Error(`Local Codex attempt ${claim.attemptId} is already active`);
9733
+ }
9734
+ this.activeAttempts.set(claim.attemptId, active);
9594
9735
  const leaseLoop = this.runLeaseLoop(active);
9595
9736
  const controlLoop = this.runControlLoop(active);
9596
9737
  try {
@@ -9662,12 +9803,14 @@ var init_runtime_worker = __esm({
9662
9803
  nativeTurnId,
9663
9804
  status: LocalCodexTurnStatusWire.Running
9664
9805
  });
9806
+ reclaimableClaimWindow.close();
9665
9807
  await this.interruptActiveTurnIfRequested(active, client);
9666
9808
  const terminal = await Promise.race([
9667
9809
  active.terminal,
9668
9810
  active.fatal.then((error) => {
9669
9811
  throw error;
9670
- })
9812
+ }),
9813
+ rejectWhenAborted(active.finished.signal)
9671
9814
  ]);
9672
9815
  await active.spoolQueue;
9673
9816
  await this.flushAllEvents(active);
@@ -9681,12 +9824,13 @@ var init_runtime_worker = __esm({
9681
9824
  });
9682
9825
  await durable.markUploaded();
9683
9826
  await this.store.deleteLocalCodexAttempt(claim.attemptId);
9684
- if (active.interruptedCleanupStarted) {
9827
+ if (active.processQuarantined) {
9685
9828
  active.finished.abort();
9686
9829
  await this.restoreAppServerAfterInterruption(active);
9687
9830
  }
9688
9831
  } catch (error) {
9689
- if (error instanceof LeaseLostError) {
9832
+ if (error instanceof WorkerStoppedError) {
9833
+ } else if (error instanceof LeaseLostError) {
9690
9834
  await this.logger.error(`Local Codex attempt ${claim.attemptId} stopped after losing its execution lease`, error);
9691
9835
  } else if (error instanceof SpoolPersistenceError) {
9692
9836
  try {
@@ -9701,17 +9845,22 @@ var init_runtime_worker = __esm({
9701
9845
  await this.failAttempt(active, error);
9702
9846
  }
9703
9847
  } finally {
9704
- this.completeInitialThreadName(active);
9705
- active.finished.abort();
9706
- await active.initialThreadName?.task;
9707
- const titleGenerationTurnId = active.initialThreadName?.generation?.turnId;
9708
- if (titleGenerationTurnId !== void 0) {
9709
- this.processPool.retireTurn(titleGenerationTurnId);
9710
- }
9711
- if (active.acceptedByAppServer && active.nativeTurnId !== void 0) {
9712
- this.processPool.retireTurn(active.nativeTurnId);
9848
+ try {
9849
+ this.completeInitialThreadName(active);
9850
+ active.finished.abort();
9851
+ await active.interruptedCleanupTask;
9852
+ await active.initialThreadName?.task;
9853
+ const titleGenerationTurnId = active.initialThreadName?.generation?.turnId;
9854
+ if (titleGenerationTurnId !== void 0) {
9855
+ this.processPool.retireTurn(titleGenerationTurnId);
9856
+ }
9857
+ if (active.acceptedByAppServer && active.nativeTurnId !== void 0) {
9858
+ this.processPool.retireTurn(active.nativeTurnId);
9859
+ }
9860
+ await Promise.all([leaseLoop, controlLoop]);
9861
+ } finally {
9862
+ this.activeAttempts.delete(claim.attemptId);
9713
9863
  }
9714
- await Promise.all([leaseLoop, controlLoop]);
9715
9864
  }
9716
9865
  }
9717
9866
  async prepareThread(client, active) {
@@ -9797,16 +9946,19 @@ var init_runtime_worker = __esm({
9797
9946
  }
9798
9947
  async runRecoveryLoop() {
9799
9948
  while (!this.lifetimeAbort.signal.aborted && !this.relay.isRevoked()) {
9800
- try {
9801
- await this.recoverPersistedAttempts();
9802
- } catch (error) {
9803
- if (!this.lifetimeAbort.signal.aborted) {
9804
- await this.logger.error(`Failed to scan persisted Local Codex attempts for connection ${this.connection.connectionId}`, error);
9805
- }
9806
- }
9949
+ await this.scanPersistedAttempts();
9807
9950
  await waitForAbort(this.lifetimeAbort.signal, recoveryRetryMilliseconds);
9808
9951
  }
9809
9952
  }
9953
+ async scanPersistedAttempts() {
9954
+ try {
9955
+ await this.recoverPersistedAttempts();
9956
+ } catch (error) {
9957
+ if (!this.lifetimeAbort.signal.aborted) {
9958
+ await this.logger.error(`Failed to scan persisted Local Codex attempts for connection ${this.connection.connectionId}`, error);
9959
+ }
9960
+ }
9961
+ }
9810
9962
  async recoverPersistedAttempts() {
9811
9963
  const attempts = await this.store.listLocalCodexAttempts();
9812
9964
  for (const persisted of attempts) {
@@ -9938,6 +10090,7 @@ var init_runtime_worker = __esm({
9938
10090
  async loseLease(active, message) {
9939
10091
  const error = new LeaseLostError(message);
9940
10092
  this.signalFatal(active, error);
10093
+ active.ignoreProcessLoss = true;
9941
10094
  await this.processPool.quarantineCurrentProcess();
9942
10095
  }
9943
10096
  async runControlLoop(active) {
@@ -10024,7 +10177,9 @@ var init_runtime_worker = __esm({
10024
10177
  if (!active.abortRequested || active.abortSent || active.nativeThreadId === void 0 || active.nativeTurnId === void 0) {
10025
10178
  return;
10026
10179
  }
10027
- active.processTreeCapture = (await this.processPool.captureCurrentProcessTree()).value;
10180
+ if (!this.processPool.hasOtherActiveExecution()) {
10181
+ active.processTreeCapture = (await this.processPool.captureCurrentProcessTree()).value;
10182
+ }
10028
10183
  await client.request("turn/interrupt", {
10029
10184
  threadId: active.nativeThreadId,
10030
10185
  turnId: active.nativeTurnId
@@ -10388,7 +10543,6 @@ ${titleInput}`,
10388
10543
  if (!this.registered) {
10389
10544
  return;
10390
10545
  }
10391
- this.processPool.release(this.connection.connectionId);
10392
10546
  this.processPool.unregister(this.connection.connectionId);
10393
10547
  this.registered = false;
10394
10548
  }
@@ -10407,18 +10561,20 @@ function localCodexProcessPoolKey(environment, nativeStoreIdentitySeed) {
10407
10561
  "connector-owned-app-server-stdio-v1"
10408
10562
  ].join("\0")).digest("hex");
10409
10563
  }
10410
- var import_node_crypto8, LocalCodexProcessPool;
10564
+ var import_node_crypto8, maximumConcurrentTurns, LocalCodexProcessPool;
10411
10565
  var init_process_pool = __esm({
10412
10566
  "../machine-connector/dist/runtime/local-codex-runtime/process-pool.js"() {
10413
10567
  "use strict";
10414
10568
  init_cjs_shims();
10415
10569
  import_node_crypto8 = require("crypto");
10416
10570
  init_app_server_client();
10571
+ maximumConcurrentTurns = 1e3;
10417
10572
  LocalCodexProcessPool = class {
10418
10573
  poolKey;
10419
10574
  logger;
10420
- activeOwner = {};
10575
+ activeSlots = /* @__PURE__ */ new Map();
10421
10576
  loadedThreadIds = /* @__PURE__ */ new Set();
10577
+ nextSlotId = 1;
10422
10578
  routes = /* @__PURE__ */ new Map();
10423
10579
  retiredTurnIds = /* @__PURE__ */ new Set();
10424
10580
  supervisor;
@@ -10428,7 +10584,7 @@ var init_process_pool = __esm({
10428
10584
  this.logger = logger;
10429
10585
  this.supervisor = new LocalCodexAppServerSupervisor(environment.HOME ?? process.cwd(), environment, nativeStoreIdentitySeed, {
10430
10586
  onNotification: async (message) => {
10431
- await this.activeRoute().value?.onNotification(message);
10587
+ await this.routeNotification(message);
10432
10588
  },
10433
10589
  onProcessExit: async (error, processGeneration) => {
10434
10590
  if (!this.supervisor.markProcessLost(processGeneration)) {
@@ -10436,18 +10592,10 @@ var init_process_pool = __esm({
10436
10592
  }
10437
10593
  this.loadedThreadIds.clear();
10438
10594
  await this.logger.error(`Shared Codex App Server generation ${processGeneration} exited`, error);
10439
- await this.activeRoute().value?.onProcessExit(error);
10595
+ await this.broadcastProcessExit(error);
10440
10596
  },
10441
10597
  onServerRequest: async (message) => {
10442
- const route = this.activeRoute().value;
10443
- if (route !== void 0) {
10444
- await route.onServerRequest(message);
10445
- return;
10446
- }
10447
- const client = this.supervisor.currentClient().value;
10448
- if (client !== void 0 && message.id !== void 0) {
10449
- await client.respondWithError(message.id, -32e3, "No owning Local Codex Runtime turn");
10450
- }
10598
+ await this.routeServerRequest(message);
10451
10599
  },
10452
10600
  onStderr: async (text3) => {
10453
10601
  await this.logger.info("Codex App Server stderr", {
@@ -10465,7 +10613,7 @@ var init_process_pool = __esm({
10465
10613
  this.routes.set(ownerId, route);
10466
10614
  }
10467
10615
  unregister(ownerId) {
10468
- if (this.activeOwner.value === ownerId) {
10616
+ if ([...this.activeSlots.values()].some((slot) => slot.ownerId === ownerId)) {
10469
10617
  throw new Error(`Local Codex route ${ownerId} still owns capacity`);
10470
10618
  }
10471
10619
  this.routes.delete(ownerId);
@@ -10474,27 +10622,35 @@ var init_process_pool = __esm({
10474
10622
  this.waitingOwners.splice(index2, 1);
10475
10623
  }
10476
10624
  }
10477
- tryAcquire(ownerId) {
10625
+ tryAcquireExecutionSlot(ownerId) {
10478
10626
  if (!this.routes.has(ownerId)) {
10479
10627
  throw new Error(`Local Codex route ${ownerId} is not registered`);
10480
10628
  }
10481
- if (this.activeOwner.value === ownerId) {
10482
- return true;
10483
- }
10484
10629
  if (!this.waitingOwners.includes(ownerId)) {
10485
10630
  this.waitingOwners.push(ownerId);
10486
10631
  }
10487
- if (this.activeOwner.value !== void 0 || this.waitingOwners[0] !== ownerId) {
10488
- return false;
10632
+ if (this.activeSlots.size >= maximumConcurrentTurns || this.waitingOwners[0] !== ownerId) {
10633
+ return {};
10489
10634
  }
10490
10635
  this.waitingOwners.shift();
10491
- this.activeOwner = { value: ownerId };
10492
- return true;
10636
+ const value = { ownerId, slotId: this.nextSlotId };
10637
+ this.nextSlotId += 1;
10638
+ this.activeSlots.set(value.slotId, { executing: false, ownerId });
10639
+ return { value };
10493
10640
  }
10494
- release(ownerId) {
10495
- if (this.activeOwner.value === ownerId) {
10496
- this.activeOwner = {};
10641
+ markExecutionStarted(slot) {
10642
+ const active = this.requireActiveSlot(slot);
10643
+ if (active.executing) {
10644
+ throw new Error(`Local Codex capacity slot ${slot.slotId} is already executing`);
10497
10645
  }
10646
+ active.executing = true;
10647
+ }
10648
+ releaseExecutionSlot(slot) {
10649
+ this.requireActiveSlot(slot);
10650
+ this.activeSlots.delete(slot.slotId);
10651
+ }
10652
+ hasOtherActiveExecution() {
10653
+ return [...this.activeSlots.values()].filter((slot) => slot.executing).length > 1;
10498
10654
  }
10499
10655
  retireTurn(nativeTurnId) {
10500
10656
  this.retiredTurnIds.add(nativeTurnId);
@@ -10539,20 +10695,82 @@ var init_process_pool = __esm({
10539
10695
  return client === void 0 ? {} : { value: await client.captureProcessTree() };
10540
10696
  }
10541
10697
  async quarantineCurrentProcess(capture) {
10698
+ const processGeneration = this.supervisor.currentClient().value?.processGeneration;
10699
+ this.loadedThreadIds.clear();
10700
+ try {
10701
+ await this.supervisor.quarantineCurrentProcess(capture);
10702
+ } finally {
10703
+ if (processGeneration !== void 0) {
10704
+ await this.broadcastProcessExit(new AppServerProcessLostError(`Shared Codex App Server generation ${processGeneration} was quarantined`));
10705
+ }
10706
+ }
10707
+ }
10708
+ async quarantineProcess(client, capture) {
10709
+ if (this.supervisor.currentClient().value === client) {
10710
+ await this.quarantineCurrentProcess(capture);
10711
+ return;
10712
+ }
10542
10713
  this.loadedThreadIds.clear();
10543
- await this.supervisor.quarantineCurrentProcess(capture);
10714
+ await client.stop(capture);
10544
10715
  }
10545
10716
  async stop() {
10546
10717
  this.loadedThreadIds.clear();
10547
10718
  await this.supervisor.stop();
10548
10719
  }
10549
- activeRoute() {
10550
- const owner = this.activeOwner.value;
10551
- if (owner === void 0) {
10552
- return {};
10720
+ matchingRoutes(message) {
10721
+ return [...this.routes.values()].filter((route) => route.ownsMessage(message));
10722
+ }
10723
+ async routeNotification(message) {
10724
+ const routes = this.matchingRoutes(message);
10725
+ if (routes.length === 0) {
10726
+ return;
10727
+ }
10728
+ if (routes.length > 1) {
10729
+ await this.logger.error("Local Codex App Server notification matched multiple runtime routes", new Error(message.method ?? "Unknown App Server notification"));
10730
+ return;
10731
+ }
10732
+ const route = routes[0];
10733
+ if (route === void 0) {
10734
+ throw new Error("Expected one Local Codex notification route");
10735
+ }
10736
+ await route.onNotification(message);
10737
+ }
10738
+ async routeServerRequest(message) {
10739
+ const routes = this.matchingRoutes(message);
10740
+ if (routes.length === 1) {
10741
+ const route = routes[0];
10742
+ if (route === void 0) {
10743
+ throw new Error("Expected one Local Codex request route");
10744
+ }
10745
+ await route.onServerRequest(message);
10746
+ return;
10747
+ }
10748
+ if (routes.length > 1) {
10749
+ await this.logger.error("Local Codex App Server request matched multiple runtime routes", new Error(message.method ?? "Unknown App Server request"));
10750
+ }
10751
+ const client = this.supervisor.currentClient().value;
10752
+ if (client !== void 0 && message.id !== void 0) {
10753
+ await client.respondWithError(message.id, -32e3, routes.length === 0 ? "No owning Local Codex Runtime turn" : "Ambiguous Local Codex Runtime turn ownership");
10754
+ }
10755
+ }
10756
+ async broadcastProcessExit(error) {
10757
+ await Promise.all([...this.routes.values()].map(async (route) => {
10758
+ try {
10759
+ await route.onProcessExit(error);
10760
+ } catch (routeError) {
10761
+ await this.logger.error("Failed to settle a Local Codex route after the shared App Server exited", routeError).catch(() => void 0);
10762
+ }
10763
+ }));
10764
+ }
10765
+ requireActiveSlot(slot) {
10766
+ const active = this.activeSlots.get(slot.slotId);
10767
+ if (active === void 0) {
10768
+ throw new Error(`Local Codex capacity slot ${slot.slotId} is not active`);
10769
+ }
10770
+ if (active.ownerId !== slot.ownerId) {
10771
+ throw new Error(`Local Codex capacity slot ${slot.slotId} belongs to another route`);
10553
10772
  }
10554
- const value = this.routes.get(owner);
10555
- return value === void 0 ? {} : { value };
10773
+ return active;
10556
10774
  }
10557
10775
  };
10558
10776
  }
@@ -14374,7 +14592,7 @@ function getApiKeyOrUndefined() {
14374
14592
 
14375
14593
  // src/utils/http.ts
14376
14594
  function getUserAgent() {
14377
- return `moxt-cli/${"0.10.0-beta.7"} (${(0, import_os.platform)()}/${(0, import_os.arch)()}) node/${process.versions.node}`;
14595
+ return `moxt-cli/${"0.10.0-beta.9"} (${(0, import_os.platform)()}/${(0, import_os.arch)()}) node/${process.versions.node}`;
14378
14596
  }
14379
14597
  async function sendRequest(method, path4, init) {
14380
14598
  return await fetch(`${getApiBaseUrl()}${path4}`, {
@@ -28796,7 +29014,7 @@ var FLUSH_TIMEOUT_MS = 3e3;
28796
29014
  var MAX_BATCH_SIZE = 100;
28797
29015
  function commonLabels() {
28798
29016
  return {
28799
- cli_version: "0.10.0-beta.7",
29017
+ cli_version: "0.10.0-beta.9",
28800
29018
  node_version: process.versions.node,
28801
29019
  os: (0, import_node_os11.platform)(),
28802
29020
  arch: (0, import_node_os11.arch)(),
@@ -28842,7 +29060,7 @@ async function flush() {
28842
29060
  }
28843
29061
  const batch = buffer.splice(0, MAX_BATCH_SIZE);
28844
29062
  const payload = {
28845
- release_id: "0.10.0-beta.7",
29063
+ release_id: "0.10.0-beta.9",
28846
29064
  client_type: "cli",
28847
29065
  metric_data_list: batch.map((m) => ({
28848
29066
  profile: "cli",
@@ -29225,9 +29443,9 @@ function commandPath3(command) {
29225
29443
  }
29226
29444
 
29227
29445
  // src/index.ts
29228
- var program2 = createProgram("0.10.0-beta.7");
29229
- installDaemonReconcileHook(program2, "0.10.0-beta.7");
29230
- installUpdateHook(program2, "0.10.0-beta.7");
29446
+ var program2 = createProgram("0.10.0-beta.9");
29447
+ installDaemonReconcileHook(program2, "0.10.0-beta.9");
29448
+ installUpdateHook(program2, "0.10.0-beta.9");
29231
29449
  instrumentProgram(program2);
29232
29450
  installExitHandler();
29233
29451
  program2.parseAsync(process.argv).then(async () => {