@cartanova/qgrid-cli 2.5.7 → 2.5.9

package/bundle/src/utils/providers/openai/codex-worker.test.ts

@@ -197,6 +197,78 @@ describe("CodexAppServerWorker active turn cleanup", () => {
   });
 });
 
+describe("CodexAppServerWorker terminal restart failure", () => {
+  it("notifies terminal failure exactly once after exhausting the restart budget", () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const onTerminalFailure = vi.fn();
+    worker.onTerminalFailure = onTerminalFailure;
+    const internals = worker as unknown as {
+      restartAttempts: number;
+      scheduleRestart(): void;
+    };
+    internals.restartAttempts = 3;
+
+    internals.scheduleRestart();
+    internals.scheduleRestart();
+
+    expect(onTerminalFailure).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not report an intentional worker shutdown as terminal failure", () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const onTerminalFailure = vi.fn();
+    worker.onTerminalFailure = onTerminalFailure;
+    const internals = worker as unknown as {
+      destroyed: boolean;
+      restartAttempts: number;
+      scheduleRestart(): void;
+    };
+    internals.destroyed = true;
+    internals.restartAttempts = 3;
+
+    internals.scheduleRestart();
+
+    expect(onTerminalFailure).not.toHaveBeenCalled();
+  });
+
+  it("resets terminal notification state after a successful initialization", async () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const request = vi.fn().mockResolvedValue({});
+    const internals = worker as unknown as {
+      rpc: { request: typeof request };
+      restartAttempts: number;
+      terminalFailureNotified: boolean;
+      spawnAndInit(): Promise<void>;
+      waitForLoginCompleted(): Promise<void>;
+    };
+    internals.restartAttempts = 3;
+    internals.terminalFailureNotified = true;
+    internals.rpc = { request };
+    vi.spyOn(internals, "spawnAndInit").mockResolvedValue(undefined);
+    vi.spyOn(internals, "waitForLoginCompleted").mockResolvedValue(undefined);
+
+    await worker.initialize();
+
+    expect(internals.restartAttempts).toBe(0);
+    expect(internals.terminalFailureNotified).toBe(false);
+  });
+});
+
 describe("CodexAppServerWorker TTFT", () => {
   it("captures fast non-stream deltas registered before turn/start completes", async () => {
     let now = 1_000;
@@ -661,6 +733,39 @@ function unsubscribedIds(request: ReturnType<typeof vi.fn>): string[] {
     .map(([, params]) => (params as { threadId: string }).threadId);
 }
 
+describe("CodexAppServerWorker partial thread creation cleanup", () => {
+  it.each([
+    ["text", false],
+    ["image", true],
+  ])("unsubscribes an untracked %s thread when history injection fails", async (_, imageGeneration) => {
+    const { worker, request } = createWorkerWithNotificationRpc(async (method) => {
+      if (method === "thread/start") {
+        return { thread: { id: "partially-created" }, model: "gpt-test" };
+      }
+      if (method === "thread/inject_items") throw new Error("history injection failed");
+      if (method === "thread/unsubscribe") return {};
+      throw new Error(`unexpected request: ${method}`);
+    });
+
+    await expect(
+      worker.createThread({
+        input: [{ type: "text", text: "current", text_elements: [] }],
+        history: [{ type: "text", text: "previous", text_elements: [] }],
+        imageGeneration,
+      }),
+    ).rejects.toThrow("history injection failed");
+
+    expect(unsubscribedIds(request)).toEqual(["partially-created"]);
+    expect(
+      (
+        worker as unknown as {
+          threadMeta: Map<string, { lastUsedAt: number }>;
+        }
+      ).threadMeta.has("partially-created"),
+    ).toBe(false);
+  });
+});
+
 describe("CodexAppServerWorker thread unsubscribe on eviction", () => {
   type ThreadMetaInternals = {
     threadMeta: Map<string, { lastUsedAt: number }>;

package/bundle/src/utils/providers/openai/codex-worker.ts

@@ -148,12 +148,14 @@ export class CodexAppServerWorker {
   private rpc: CodexRpcClient | null = null;
   private codexHome: string;
   private restartAttempts = 0;
+  private terminalFailureNotified = false;
   private ready = false;
   private destroyed = false;
   private busy = false;
   private activeTurnAbort?: ActiveTurnAbort;
   active = true;
   onReady?: () => void;
+  onTerminalFailure?: () => void;
   // restart 시 rpc 객체가 새로 생성되므로, 핸들러를 보관했다가 매 spawn마다 재바인딩한다.
   serverRequestHandler?: (method: string, params: unknown) => Promise<unknown>;
 
@@ -246,6 +248,7 @@ export class CodexAppServerWorker {
     await this.waitForLoginCompleted();
     this.ready = true;
     this.restartAttempts = 0;
+    this.terminalFailureNotified = false;
   }
 
   async startBrowserLogin(): Promise<string> {
@@ -261,6 +264,7 @@ export class CodexAppServerWorker {
     await this.waitForLoginCompleted();
     this.ready = true;
     this.restartAttempts = 0;
+    this.terminalFailureNotified = false;
     logger.info(`worker ${this.config.tokenName} browser login completed`);
   }
 
@@ -453,20 +457,28 @@ export class CodexAppServerWorker {
     );
 
     const threadId = thread.id;
-    if (req.history?.length) {
-      await this.rpc.request("thread/inject_items", {
-        threadId,
-        items: req.history,
-      } satisfies ThreadInjectItemsParams);
-    }
-    // 이미지 turn 은 재사용에서 제외(R8): base64 가 thread 메모리·캐시 prefix 에
-    // 상주하는 것을 막고, 이미지 thread 를 1 회용으로 둔다. threadMeta 미등록 →
-    // 후속 reuse 조회에서 이 thread 는 후보가 되지 않는다.
-    if (!req.imageGeneration) {
-      this.threadMeta.set(threadId, { lastUsedAt: Date.now() });
-    }
+    try {
+      if (req.history?.length) {
+        await this.rpc.request("thread/inject_items", {
+          threadId,
+          items: req.history,
+        } satisfies ThreadInjectItemsParams);
+      }
+      // 이미지 turn 은 재사용에서 제외(R8): base64 가 thread 메모리·캐시 prefix 에
+      // 상주하는 것을 막고, 이미지 thread 를 1 회용으로 둔다. threadMeta 미등록 →
+      // 후속 reuse 조회에서 이 thread 는 후보가 되지 않는다.
+      if (!req.imageGeneration) {
+        this.threadMeta.set(threadId, { lastUsedAt: Date.now() });
+      }
 
-    return { threadId, model: req.model ?? threadModel };
+      return { threadId, model: req.model ?? threadModel };
+    } catch (error) {
+      // thread/start 는 현재 RPC 커넥션을 구독자로 등록한다. 생성 후 초기화가
+      // 끝나기 전에 실패하면 executeTurn 의 finally 와 lazy sweep 어느 쪽도 이
+      // thread 를 볼 수 없으므로 여기서 즉시 재사용 차단과 구독 해제를 함께 한다.
+      this.evictThread(threadId);
+      throw error;
+    }
   }
 
   // 기존 thread 에 turn 만 실행 (inject 없음). conversation_id 유지 → prompt cache 적중.
@@ -865,9 +877,18 @@ export class CodexAppServerWorker {
   // ── Restart ─────────────────────────────────────────────────────
 
   private scheduleRestart(): void {
-    if (this.destroyed || this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
-      if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
+    if (this.destroyed) return;
+    if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
+      if (!this.terminalFailureNotified) {
+        this.terminalFailureNotified = true;
         logger.warn(`worker ${this.config.tokenName} max restart attempts reached`);
+        try {
+          this.onTerminalFailure?.();
+        } catch (e) {
+          logger.warn(
+            `worker ${this.config.tokenName} terminal failure callback failed: ${(e as Error).message}`,
+          );
+        }
       }
       return;
     }