@cartanova/qgrid-cli 2.5.8 → 2.5.10

package/bundle/src/utils/providers/openai/codex-worker.test.ts

@@ -197,6 +197,78 @@ describe("CodexAppServerWorker active turn cleanup", () => {
   });
 });
 
+describe("CodexAppServerWorker terminal restart failure", () => {
+  it("notifies terminal failure exactly once after exhausting the restart budget", () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const onTerminalFailure = vi.fn();
+    worker.onTerminalFailure = onTerminalFailure;
+    const internals = worker as unknown as {
+      restartAttempts: number;
+      scheduleRestart(): void;
+    };
+    internals.restartAttempts = 3;
+
+    internals.scheduleRestart();
+    internals.scheduleRestart();
+
+    expect(onTerminalFailure).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not report an intentional worker shutdown as terminal failure", () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const onTerminalFailure = vi.fn();
+    worker.onTerminalFailure = onTerminalFailure;
+    const internals = worker as unknown as {
+      destroyed: boolean;
+      restartAttempts: number;
+      scheduleRestart(): void;
+    };
+    internals.destroyed = true;
+    internals.restartAttempts = 3;
+
+    internals.scheduleRestart();
+
+    expect(onTerminalFailure).not.toHaveBeenCalled();
+  });
+
+  it("resets terminal notification state after a successful initialization", async () => {
+    const worker = new CodexAppServerWorker({
+      tokenId: 1,
+      tokenName: "token",
+      accessToken: "access",
+      accountId: "account",
+    });
+    const request = vi.fn().mockResolvedValue({});
+    const internals = worker as unknown as {
+      rpc: { request: typeof request };
+      restartAttempts: number;
+      terminalFailureNotified: boolean;
+      spawnAndInit(): Promise<void>;
+      waitForLoginCompleted(): Promise<void>;
+    };
+    internals.restartAttempts = 3;
+    internals.terminalFailureNotified = true;
+    internals.rpc = { request };
+    vi.spyOn(internals, "spawnAndInit").mockResolvedValue(undefined);
+    vi.spyOn(internals, "waitForLoginCompleted").mockResolvedValue(undefined);
+
+    await worker.initialize();
+
+    expect(internals.restartAttempts).toBe(0);
+    expect(internals.terminalFailureNotified).toBe(false);
+  });
+});
+
 describe("CodexAppServerWorker TTFT", () => {
   it("captures fast non-stream deltas registered before turn/start completes", async () => {
     let now = 1_000;
@@ -310,6 +382,92 @@ describe("CodexAppServerWorker TTFT", () => {
   });
 });
 
+describe("CodexAppServerWorker thread activity timestamps", () => {
+  type ThreadMetaInternals = {
+    threadMeta: Map<string, { lastUsedAt: number }>;
+  };
+
+  it("records non-stream thread activity at turn completion", async () => {
+    let now = 1_000;
+    const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => now);
+    const { worker, request, handlers } = createWorkerWithNotificationRpc(
+      async (method, _params) => {
+        if (method === "thread/start") {
+          return { thread: { id: "thread-1" }, model: "gpt-test" };
+        }
+        if (method === "turn/start") return { turn: { id: "turn-1" } };
+        throw new Error(`unexpected request: ${method}`);
+      },
+    );
+
+    try {
+      const turn = worker.executeTurn({
+        input: [{ type: "text", text: "hi", text_elements: [] }],
+      });
+      await vi.waitFor(() =>
+        expect(request).toHaveBeenCalledWith("turn/start", expect.anything()),
+      );
+      now = 700_000;
+      handlers.get("turn/completed")?.({
+        threadId: "thread-1",
+        turn: { status: "completed", durationMs: 699_000 },
+      } as never);
+
+      await turn;
+
+      expect(
+        (worker as unknown as ThreadMetaInternals).threadMeta.get("thread-1")?.lastUsedAt,
+      ).toBe(700_000);
+    } finally {
+      nowSpy.mockRestore();
+    }
+  });
+
+  it("records streaming thread activity at turn completion", async () => {
+    let now = 2_000;
+    const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => now);
+    const { worker, request, handlers } = createWorkerWithNotificationRpc(
+      async (method, _params) => {
+        if (method === "thread/start") {
+          return { thread: { id: "thread-1" }, model: "gpt-test" };
+        }
+        if (method === "turn/start") return { turn: { id: "turn-1" } };
+        throw new Error(`unexpected request: ${method}`);
+      },
+    );
+    const callbacks: StreamCallbacks = {
+      onDelta: vi.fn(),
+      onComplete: vi.fn(),
+      onError: vi.fn(),
+    };
+
+    try {
+      const turn = worker.executeTurnStream(
+        {
+          input: [{ type: "text", text: "hi", text_elements: [] }],
+        },
+        callbacks,
+      );
+      await vi.waitFor(() =>
+        expect(request).toHaveBeenCalledWith("turn/start", expect.anything()),
+      );
+      now = 800_000;
+      handlers.get("turn/completed")?.({
+        threadId: "thread-1",
+        turn: { status: "completed", durationMs: 798_000 },
+      } as never);
+
+      await turn;
+
+      expect(
+        (worker as unknown as ThreadMetaInternals).threadMeta.get("thread-1")?.lastUsedAt,
+      ).toBe(800_000);
+    } finally {
+      nowSpy.mockRestore();
+    }
+  });
+});
+
 describe("CodexAppServerWorker prompt prefix", () => {
   it("uses the same base instructions with and without an output schema", async () => {
     type WorkerInternals = {
@@ -728,7 +886,7 @@ describe("CodexAppServerWorker thread unsubscribe on eviction", () => {
     expect(unsubscribedIds(request).toSorted()).toEqual(["stale-1", "stale-2"]);
   });
 
-  it("sends thread/unsubscribe for LRU-evicted threads over the cap", () => {
+  it("caps periodic LRU cleanup at the configured maximum", () => {
     const { worker, request, meta } = createWorkerWithUnsubscribeSpy();
     const now = Date.now();
     for (let i = 0; i < 17; i++) {
@@ -737,7 +895,19 @@ describe("CodexAppServerWorker thread unsubscribe on eviction", () => {
 
     worker.sweepIdleThreads();
 
-    // 상한(MAX-1=15)으로 줄이며 가장 오래된 2개가 축출·구독 해제된다.
+    expect(meta.size).toBe(16);
+    expect(unsubscribedIds(request)).toEqual(["t-0"]);
+  });
+
+  it("reserves one LRU slot before creating a new thread", () => {
+    const { worker, request, meta } = createWorkerWithUnsubscribeSpy();
+    const now = Date.now();
+    for (let i = 0; i < 17; i++) {
+      meta.set(`t-${i}`, { lastUsedAt: now - (17 - i) * 1_000 });
+    }
+
+    worker.sweepIdleThreads(true);
+
     expect(meta.size).toBe(15);
     expect(unsubscribedIds(request).toSorted()).toEqual(["t-0", "t-1"]);
   });

package/bundle/src/utils/providers/openai/codex-worker.ts

@@ -148,12 +148,14 @@ export class CodexAppServerWorker {
   private rpc: CodexRpcClient | null = null;
   private codexHome: string;
   private restartAttempts = 0;
+  private terminalFailureNotified = false;
   private ready = false;
   private destroyed = false;
   private busy = false;
   private activeTurnAbort?: ActiveTurnAbort;
   active = true;
   onReady?: () => void;
+  onTerminalFailure?: () => void;
   // restart 시 rpc 객체가 새로 생성되므로, 핸들러를 보관했다가 매 spawn마다 재바인딩한다.
   serverRequestHandler?: (method: string, params: unknown) => Promise<unknown>;
 
@@ -246,6 +248,7 @@ export class CodexAppServerWorker {
     await this.waitForLoginCompleted();
     this.ready = true;
     this.restartAttempts = 0;
+    this.terminalFailureNotified = false;
   }
 
   async startBrowserLogin(): Promise<string> {
@@ -261,6 +264,7 @@ export class CodexAppServerWorker {
     await this.waitForLoginCompleted();
     this.ready = true;
     this.restartAttempts = 0;
+    this.terminalFailureNotified = false;
     logger.info(`worker ${this.config.tokenName} browser login completed`);
   }
 
@@ -427,7 +431,7 @@ export class CodexAppServerWorker {
   // thread 의 기본 model 도 반환 (req.model 미지정 시 fallback).
   async createThread(req: TurnRequest): Promise<{ threadId: string; model: string }> {
     if (!this.rpc || !this.ready) throw new Error("worker not ready");
-    this.sweepIdleThreads();
+    this.sweepIdleThreads(true);
 
     const threadConfig: ThreadStartParams["config"] = {
       ...THREAD_DEFAULTS.config,
@@ -524,7 +528,11 @@ export class CodexAppServerWorker {
       // 이미지 thread 는 1 회용 cold thread 인데 threadMeta 에 등록되지 않아
       // sweep 이 못 잡는다. MB 급 base64 가 상주하므로 turn 종료 즉시(성공/실패
       // 불문) 구독을 끊어 codex 30분 언로드 대상으로 만든다.
-      if (req.imageGeneration) this.unsubscribeThread(threadId);
+      if (req.imageGeneration) {
+        this.unsubscribeThread(threadId);
+      } else {
+        this.touchThread(threadId);
+      }
     }
   }
 
@@ -544,18 +552,22 @@ export class CodexAppServerWorker {
     cb.onThreadId?.(threadId);
     const ttftTracker = createTtftTracker();
     const resultPromise = this.consumeStreamNotifications(threadId, model, cb, ttftTracker);
-    let turnId: string;
     try {
-      ttftTracker.markStart();
-      ({ turnId } = await this.startTurnOnThread(threadId, req));
-    } catch (e) {
-      this.failActiveTurn(e as Error);
-      await resultPromise.catch(() => {});
-      throw e;
+      let turnId: string;
+      try {
+        ttftTracker.markStart();
+        ({ turnId } = await this.startTurnOnThread(threadId, req));
+      } catch (e) {
+        this.failActiveTurn(e as Error);
+        await resultPromise.catch(() => {});
+        throw e;
+      }
+      cb.onTurnId?.(turnId);
+      await resultPromise;
+      return { threadId };
+    } finally {
+      this.touchThread(threadId);
     }
-    cb.onTurnId?.(turnId);
-    await resultPromise;
-    return { threadId };
   }
 
   // ── thread 재사용 지원 ───────────────────────────────────────────
@@ -568,21 +580,21 @@ export class CodexAppServerWorker {
     return this.threadMeta.has(threadId);
   }
 
-  // idle TTL 초과 / 개수 상한 초과 thread 를 정리. createThread 직전 lazy 호출.
+  // idle TTL 초과 / 개수 상한 초과 thread 를 정리.
   // 맵 제거(재사용 라우팅 차단)와 함께 thread/unsubscribe 를 보낸다 — codex 는
   // "구독자 0 + idle" 이 30분(THREAD_UNLOADING_DELAY, 하드코딩) 지속된 thread 를
   // 메모리에서 자동 언로드하는데, thread/start 가 이 커넥션을 자동 구독자로 등록해
   // 두기 때문에 unsubscribe 없이는 그 언로드가 영원히 발동하지 않는다(SON-516 실증).
-  sweepIdleThreads(): void {
+  sweepIdleThreads(reserveSlotForNewThread = false): void {
     const now = Date.now();
     for (const [id, meta] of this.threadMeta) {
       if (now - meta.lastUsedAt > CodexAppServerWorker.THREAD_IDLE_TTL_MS) {
         this.evictThread(id);
       }
     }
-    // createThread 직전 호출 → 새 thread 1 개가 곧 추가된다. 추가 후에도 MAX 를 넘지 않도록
-    // MAX-1 이하로 줄여 둔다(그래야 생성 직후 정확히 MAX). LRU: lastUsedAt 오름차순으로 제거.
-    const limit = CodexAppServerWorker.MAX_THREADS_PER_WORKER - 1;
+    // createThread 직전에는 새 thread 1개가 곧 추가되므로 MAX-1을 사용한다.
+    // 주기 sweep은 현재 cap인 MAX까지만 줄인다. LRU: lastUsedAt 오름차순으로 제거.
+    const limit = CodexAppServerWorker.MAX_THREADS_PER_WORKER - (reserveSlotForNewThread ? 1 : 0);
     if (this.threadMeta.size > limit) {
       const sorted = [...this.threadMeta.entries()].toSorted(
         (a, b) => a[1].lastUsedAt - b[1].lastUsedAt,
@@ -592,6 +604,11 @@ export class CodexAppServerWorker {
     }
   }
 
+  private touchThread(threadId: string): void {
+    const meta = this.threadMeta.get(threadId);
+    if (meta) meta.lastUsedAt = Date.now();
+  }
+
   // thread 재사용 포기: 맵 제거(라우팅 차단)와 구독 해제를 한 단위로 묶는다.
   // unsubscribe 한 thread 에 turn 을 보내면 알림이 이 커넥션으로 오지 않으므로
   // 두 동작은 반드시 함께여야 한다.
@@ -873,9 +890,18 @@ export class CodexAppServerWorker {
   // ── Restart ─────────────────────────────────────────────────────
 
   private scheduleRestart(): void {
-    if (this.destroyed || this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
-      if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
+    if (this.destroyed) return;
+    if (this.restartAttempts >= MAX_RESTART_ATTEMPTS) {
+      if (!this.terminalFailureNotified) {
+        this.terminalFailureNotified = true;
         logger.warn(`worker ${this.config.tokenName} max restart attempts reached`);
+        try {
+          this.onTerminalFailure?.();
+        } catch (e) {
+          logger.warn(
+            `worker ${this.config.tokenName} terminal failure callback failed: ${(e as Error).message}`,
+          );
+        }
       }
       return;
     }