@covro/covro 0.5.1-dev.2 → 0.5.1-dev.3

package/dist/server/service.js

@@ -1,3 +1,5 @@
+import { CapacityRetries } from "./capacity-retry.js";
+import { isModelCapacityError } from "../shared/turn-errors.js";
 import { attentionKey, pendingAttention } from "../shared/thread-attention.js";
 import { ShellService } from "./shell.js";
 import { readGithubRepository } from "./github-repository.js";
@@ -49,11 +51,16 @@ class GatewayService {
     this.shell = new ShellService(this.codex);
     this.events = new EventHub(options.limits?.eventBytes);
     this.projection = new Projection(options.limits?.itemBytes);
+    this.capacityRetries = new CapacityRetries(
+      (job) => this.dispatchCapacityRetry(job),
+      (id) => this.publishThread(id)
+    );
     this.currentEpoch = this.codex.epoch;
     this.store.recover();
     this.removeListener = this.codex.onEvent((event) => this.consume(event));
   }
   options;
+  capacityRetries;
   projectClones;
   shell;
   events;
@@ -86,6 +93,7 @@ class GatewayService {
   runtimeSettingsUpdates = /* @__PURE__ */ new Map();
   inFlightOperations = /* @__PURE__ */ new Set();
   pendingThreadCreations = /* @__PURE__ */ new Map();
+  capacityLinks = /* @__PURE__ */ new Map();
   listProjects() {
     return this.projects.list();
   }
@@ -132,6 +140,8 @@ class GatewayService {
     return this.maintenanceReason === "backup" ? new AppError("BACKUP_MAINTENANCE", "\u6B63\u5728\u5907\u4EFD\uFF0C\u8BF7\u5B8C\u6210\u540E\u91CD\u8BD5\u3002", 503) : new AppError("UPDATE_MAINTENANCE", "\u6B63\u5728\u5347\u7EA7\uFF0C\u8BF7\u7B49\u5F85\u670D\u52A1\u91CD\u65B0\u8FDE\u63A5", 503);
   }
   prepareUpdate(reason = "update") {
+    if (this.capacityRetries.active())
+      return { ready: false, reason: "\u7B49\u5F85\u6A21\u578B\u5BB9\u91CF\u91CD\u8BD5\u7ED3\u675F\u6216\u53D6\u6D88" };
     if (this.projectClones.hasActive) return { ready: false, reason: "\u7B49\u5F85\u9879\u76EE\u514B\u9686\u5B8C\u6210\u6216\u53D6\u6D88" };
     if (this.shell.active()) return { ready: false, reason: "\u7B49\u5F85 Shell \u547D\u4EE4\u7ED3\u675F" };
     if (this.inFlightOperations.size || this.resumes.size || this.pendingThreadCreations.size)
@@ -179,7 +189,7 @@ class GatewayService {
       return link ? link.projectId === projectId : entry.workspaceKey === project.workspaceKey;
     });
     const active = this.projection.values().find(
-      (thread) => thread.projectId === projectId && (isActive(thread.status) || thread.actions.some((action) => ["pending", "responding"].includes(action.status)))
+      (thread) => thread.projectId === projectId && (isActive(thread.status) || this.capacityRetries.pending(thread.id) || thread.actions.some((action) => ["pending", "responding"].includes(action.status)))
     );
     if (pending || marker || active || this.shell.active(projectId))
       throw new AppError(
@@ -453,6 +463,7 @@ class GatewayService {
   updateArchiveState(threadId, archived) {
     const thread = this.projection.get(threadId);
     if (!thread || Boolean(thread.archived) === archived) return;
+    if (archived) this.capacityRetries.clear(threadId);
     thread.archived = archived;
     this.archiveRevisions.set(threadId, (this.archiveRevisions.get(threadId) ?? 0) + 1);
     if (archived) this.releaseThreadOwnership(threadId);
@@ -494,6 +505,11 @@ class GatewayService {
       throw new AppError("HISTORY_UNAVAILABLE", "\u5F53\u524D Codex \u4E0D\u652F\u6301\u5386\u53F2\u5206\u9875", 409);
     const page = this.normalize(await this.codex.readThreadPage(threadId, cursor));
     this.workspaces.assertOwnership(page.projectId, page.cwd, page.id);
+    const links = this.retryLinks(threadId);
+    for (const turn of page.turns) {
+      const root = links.get(turn.id);
+      if (root) turn.retryOf = root;
+    }
     return page;
   }
   async exportThread(threadId) {
@@ -620,6 +636,7 @@ class GatewayService {
   }
   async resumeThread(threadId, input) {
     return this.perform("thread.resume", threadId, input, async () => {
+      this.capacityRetries.clear(threadId);
       const thread = await this.loadThread(threadId);
       this.ensureReady();
       this.ensureArchiveAvailable(thread);
@@ -721,11 +738,21 @@ class GatewayService {
     });
   }
   async startTurn(threadId, input) {
+    return this.dispatchTurn(threadId, input);
+  }
+  async dispatchTurn(threadId, input, retry, expectedTurnId) {
     return this.perform("turn.start", threadId, input, async () => {
+      if (!retry && !expectedTurnId && this.capacityRetries.view(threadId)?.status === "waiting")
+        this.capacityRetries.clear(threadId);
       const thread = await this.loadThread(threadId);
       this.ensureManaged(thread);
       const ownership = this.workspaces.assertOwnership(thread.projectId, thread.cwd, thread.id);
-      if (isActive(thread.status))
+      if (expectedTurnId) {
+        const last = thread.turns.at(-1);
+        if (last?.id !== expectedTurnId || last.status !== "failed" || !isModelCapacityError(last.error ?? "", last.errorCode))
+          throw new AppError("TURN_CONFLICT", "\u76EE\u6807\u5BB9\u91CF\u5931\u8D25\u5DF2\u6539\u53D8\uFF0C\u8BF7\u5237\u65B0\u540E\u7EE7\u7EED", 409);
+      }
+      if (isActive(this.projection.get(threadId)?.status ?? thread.status))
         throw new AppError("THREAD_BUSY", "\u4F1A\u8BDD\u5DF2\u6709\u6D3B\u52A8\u4EFB\u52A1\uFF0C\u8BF7\u8FFD\u52A0\u8981\u6C42\u6216\u4E2D\u65AD", 409, { threadId });
       this.ensureCompatibilityConfirmed();
       this.store.dispatch(input.clientOperationId, {
@@ -733,6 +760,15 @@ class GatewayService {
         workspaceKey: ownership.workspaceKey,
         codexEpoch: this.codex.epoch
       });
+      if (retry && !this.capacityRetries.owns(retry))
+        throw new AppError("TURN_CONFLICT", "\u5BB9\u91CF\u91CD\u8BD5\u5DF2\u53D6\u6D88\u6216\u88AB\u66FF\u4EE3", 409);
+      const candidate = retry ?? this.capacityRetries.begin(
+        threadId,
+        input.clientOperationId,
+        this.retrySettings(thread.runtimeSettings, input.settings),
+        this.codex.epoch
+      );
+      const retryOf = retry?.rootTurnId ?? (expectedTurnId ? this.retryLinks(threadId).get(expectedTurnId) ?? expectedTurnId : void 0);
       this.turnBaselines.set(input.clientOperationId, new Set(thread.turns.map((turn2) => turn2.id)));
       this.publishThread(threadId);
       const revision = this.projection.revision(threadId);
@@ -746,15 +782,70 @@ class GatewayService {
       this.store.attachTurn(input.clientOperationId, turn.id);
       this.turnBaselines.delete(input.clientOperationId);
       this.projection.turn(threadId, turn, this.projection.revision(threadId) !== revision);
+      this.capacityRetries.observe({ type: "turn", threadId, turn }, input.clientOperationId);
+      if (retryOf) this.retryLinks(threadId).set(turn.id, retryOf);
+      if (candidate.cancelled && retry) await this.interruptCancelledCapacity(candidate, turn.id);
       const projectedTurn = this.projection.get(threadId)?.turns.find((entry) => entry.id === turn.id);
       if (projectedTurn && isTerminal(projectedTurn.status))
         this.store.finishTurn(threadId, turn.id, this.codex.epoch);
       this.publishThread(threadId);
-      return { result: { threadId, turnId: turn.id }, turnId: turn.id };
+      return {
+        result: { threadId, turnId: turn.id, ...retryOf ? { retryOf } : {} },
+        turnId: turn.id
+      };
     });
   }
+  retrySettings(runtime, input = {}) {
+    return {
+      ...runtime?.model ? { model: runtime.model } : {},
+      ...runtime?.reasoningEffort ? { reasoningEffort: runtime.reasoningEffort } : {},
+      ...runtime?.serviceTier === "fast" || runtime?.serviceTier === null ? { serviceTier: runtime.serviceTier } : {},
+      ...input
+    };
+  }
+  async dispatchCapacityRetry(job) {
+    if (!this.capacityRetries.owns(job) || !this.accepting || this.maintenance || !this.codex.ready || this.codex.epoch !== job.epoch)
+      throw new AppError("TURN_CONFLICT", "\u5BB9\u91CF\u91CD\u8BD5\u7684\u8FD0\u884C\u73AF\u5883\u5DF2\u6539\u53D8", 409);
+    const thread = this.projection.get(job.threadId);
+    const last = thread?.turns.at(-1);
+    if (!thread || !last || last.id !== job.turnId || last.status !== "failed" || this.store.activeForThread(job.threadId) || !job.previousOperationId || this.store.get(job.previousOperationId)?.state !== "acknowledged" || this.inFlightOperations.has(job.previousOperationId))
+      throw new AppError("TURN_CONFLICT", "\u539F\u4EFB\u52A1\u72B6\u6001\u5DF2\u6539\u53D8\uFF0C\u8BF7\u68C0\u67E5\u540E\u7EE7\u7EED", 409);
+    await this.dispatchTurn(
+      job.threadId,
+      {
+        clientOperationId: job.operationId,
+        text: "",
+        settings: job.settings
+      },
+      job
+    );
+  }
+  async cancelCapacityRetry(threadId, input) {
+    return this.perform("turn.retry.cancel", threadId, input, async () => {
+      const thread = await this.loadThread(threadId);
+      this.ensureManaged(thread);
+      const job = this.capacityRetries.current(threadId);
+      if (!job?.view || job.rootTurnId !== input.expectedTurnId)
+        throw new AppError("TURN_CONFLICT", "\u5BB9\u91CF\u91CD\u8BD5\u5DF2\u7ED3\u675F\u6216\u88AB\u66FF\u4EE3", 409);
+      this.store.dispatch(input.clientOperationId);
+      this.capacityRetries.cancel(job);
+      if (job.attempt > 0 && job.turnId) await this.interruptCancelledCapacity(job, job.turnId);
+      return { result: { threadId, cancelled: true } };
+    });
+  }
+  async interruptCancelledCapacity(job, turnId) {
+    const turn = this.projection.get(job.threadId)?.turns.find((entry) => entry.id === turnId);
+    if (!job.cancelled || job.interruptSent || !turn || !isActive(turn.status) || this.codex.epoch !== job.epoch || !this.codex.ready)
+      return;
+    job.interruptSent = true;
+    await this.codex.interruptTurn(job.threadId, turnId);
+  }
+  async retryTurn(threadId, input) {
+    return this.dispatchTurn(threadId, { ...input, text: "" }, void 0, input.expectedTurnId);
+  }
   async compactThread(threadId, input) {
     return this.perform("thread.compact", threadId, input, async () => {
+      this.capacityRetries.clear(threadId);
       const thread = await this.loadThread(threadId);
       this.ensureManaged(thread);
       const ownership = this.workspaces.assertOwnership(thread.projectId, thread.cwd, thread.id);
@@ -779,6 +870,7 @@ class GatewayService {
   }
   async startReview(threadId, input) {
     return this.perform("review.start", threadId, input, async () => {
+      this.capacityRetries.clear(threadId);
       const thread = await this.loadThread(threadId);
       this.ensureManaged(thread);
       const ownership = this.workspaces.assertOwnership(thread.projectId, thread.cwd, thread.id);
@@ -916,6 +1008,7 @@ class GatewayService {
   }
   async shutdown() {
     this.accepting = false;
+    this.capacityRetries.clearAll();
     await this.projectClones.close();
     this.store.recover();
     this.projection.runtimeReset();
@@ -957,6 +1050,8 @@ class GatewayService {
         const action = this.projection.resolveAction(target, "pending");
         if (action?.threadId) this.publishThread(action.threadId);
       }
+      if (kind === "turn.start")
+        this.capacityRetries.dispatchFailed(target, input.clientOperationId);
       if (dispatched && kind === "turn.start") this.publishThread(target);
       this.events.publish("operation", operation);
       if (uncertain)
@@ -1190,8 +1285,25 @@ class GatewayService {
       thread.hasMessages || thread.archived || thread.turns.length || this.store.activeForThread(thread.id) || isActive(thread.status) || thread.actions.some((action) => ["pending", "responding"].includes(action.status))
     );
   }
+  retryLinks(threadId) {
+    let links = this.capacityLinks.get(threadId);
+    if (!links) {
+      links = this.store.capacityRetryTurns(threadId);
+      this.capacityLinks.set(threadId, links);
+    }
+    return links;
+  }
   threadView(thread) {
     const result = structuredClone(thread);
+    const links = this.capacityLinks.get(thread.id) ?? (thread.turns.some((turn) => !turn.items.some((item) => item.type === "userMessage")) ? this.retryLinks(thread.id) : void 0);
+    for (const turn of result.turns) {
+      const root = links?.get(turn.id);
+      if (root) turn.retryOf = root;
+    }
+    const retry = this.capacityRetries.view(thread.id);
+    if (retry) result.capacityRetry = structuredClone(retry);
+    else delete result.capacityRetry;
+    if (this.capacityRetries.pending(thread.id)) result.status = "running";
     result.archived = thread.archived ?? false;
     result.historyVisible = this.historyVisible(thread);
     const marker = this.store.activeForThread(thread.id);
@@ -1240,6 +1352,17 @@ class GatewayService {
     }
   }
   consume(event) {
+    if (event.type === "runtime" && (!event.diagnostics.ready || event.diagnostics.codexEpoch !== this.currentEpoch))
+      this.capacityRetries.clearAll();
+    const eventThread = event.type === "action" ? event.action.threadId : "threadId" in event ? event.threadId : void 0;
+    const eventMarker = eventThread ? this.store.activeForThread(eventThread) : void 0;
+    const matchesDispatch = event.type !== "turn" || eventMarker?.turnId === event.turn.id || eventMarker?.turnId === null && this.turnBaselines.get(eventMarker.operationId)?.has(event.turn.id) === false;
+    this.capacityRetries.observe(event, matchesDispatch ? eventMarker?.operationId : void 0);
+    if (event.type === "turn") {
+      const job = this.capacityRetries.current(event.threadId);
+      if (job?.attempt && job.turnId === event.turn.id && job.rootTurnId)
+        this.retryLinks(event.threadId).set(event.turn.id, job.rootTurnId);
+    }
     if (event.type === "runtime") {
       this.compatibilityConfirmation(event.diagnostics);
       if (!event.diagnostics.ready || event.diagnostics.codexEpoch !== this.currentEpoch) {
@@ -1304,7 +1427,13 @@ class GatewayService {
       return;
     }
     if (event.type === "turnError" || event.type === "turnProgress") {
-      const changed = event.type === "turnError" ? this.projection.turnError(thread.id, event.turnId, event.message, event.willRetry) : this.projection.turnProgress(thread.id, event.turnId);
+      const changed = event.type === "turnError" ? this.projection.turnError(
+        thread.id,
+        event.turnId,
+        event.message,
+        event.willRetry,
+        event
+      ) : this.projection.turnProgress(thread.id, event.turnId);
       if (changed) this.publishThread(thread.id);
       return;
     }
@@ -1338,7 +1467,7 @@ class GatewayService {
       if (isTerminal(event.turn.status)) {
         this.store.finishTurn(thread.id, event.turn.id, this.codex.epoch);
         this.projection.expireTurn(thread.id, event.turn.id);
-        if (newCompletion) {
+        if (newCompletion && !this.capacityRetries.pending(thread.id)) {
           try {
             this.workspaces.assertOwnership(thread.projectId, thread.cwd, thread.id);
             this.events.publishCompletion({