@triflux/core 10.1.0 → 10.25.1

package/hub/cli-adapter-base.mjs

@@ -1,10 +1,66 @@
 // hub/cli-adapter-base.mjs — codex/gemini 공통 CLI adapter 인터페이스
 // Phase 2: codex-adapter.mjs에서 추출한 재사용 가능 유틸리티
 
-import { execSync, spawn } from 'node:child_process';
-import { existsSync, readFileSync } from 'node:fs';
+import { execSync, spawn } from "node:child_process";
+import { existsSync, readFileSync, statSync } from "node:fs";
 
-import { killProcess, IS_WINDOWS } from './platform.mjs';
+import { writePromptToTmpFile } from "./lib/prompt-tmp.mjs";
+import { IS_WINDOWS, killProcess } from "./platform.mjs";
+
+// ── Quota retry-after 파싱 ──────────────────────────────────────
+
+const FALLBACK_COOLDOWN_MS = { codex: 5 * 3600_000, gemini: 86400_000 };
+
+/**
+ * 에러 텍스트에서 retry-after 시간을 동적 파싱.
+ * "try again at 2026-04-11T10:00:00Z", "retry after 3600 seconds",
+ * "resets in 7 days", "wait 5 hours" 등을 감지.
+ */
+export function parseRetryAfterMs(text, provider) {
+  // 1. ISO timestamp: "try again at 2026-04-11T10:00:00"
+  const isoMatch = text.match(
+    /(?:try again|retry|available|resets?)\s+(?:at|after)\s+(\d{4}-\d{2}-\d{2}T[\d:.]+Z?)/i,
+  );
+  if (isoMatch) {
+    const target = new Date(isoMatch[1]).getTime();
+    if (target > Date.now()) return target - Date.now();
+  }
+
+  // 2. Duration: "N seconds/minutes/hours/days"
+  const durMatch = text.match(
+    /(?:retry|wait|resets?|again)\s+(?:in\s+|after\s+)?(\d+)\s*(second|minute|hour|day|week)/i,
+  );
+  if (durMatch) {
+    const n = Number(durMatch[1]);
+    const unit = durMatch[2].toLowerCase();
+    const multipliers = {
+      second: 1000,
+      minute: 60_000,
+      hour: 3600_000,
+      day: 86400_000,
+      week: 604800_000,
+    };
+    return n * (multipliers[unit] || 3600_000);
+  }
+
+  // 3. "7 days" / "24 hours" 단독 (context 없이)
+  const standaloneMatch = text.match(/(\d+)\s*(day|hour|week|minute)/i);
+  if (standaloneMatch) {
+    const n = Number(standaloneMatch[1]);
+    const unit = standaloneMatch[2].toLowerCase();
+    const multipliers = {
+      minute: 60_000,
+      hour: 3600_000,
+      day: 86400_000,
+      week: 604800_000,
+    };
+    const parsed = n * (multipliers[unit] || 3600_000);
+    if (parsed >= 3600_000) return parsed; // 1시간 이상만 신뢰
+  }
+
+  // 4. fallback: provider 기본값
+  return FALLBACK_COOLDOWN_MS[provider] || 5 * 3600_000;
+}
 
 // ── Codex CLI compatibility ─────────────────────────────────────
 
@@ -17,12 +73,23 @@ let _cachedVersion = null;
  */
 export function getCodexVersion() {
   if (_cachedVersion !== null) return _cachedVersion;
+  const override = Number(process.env.TFX_CODEX_VERSION_MINOR);
+  if (Number.isFinite(override) && override > 0) {
+    _cachedVersion = override;
+    return _cachedVersion;
+  }
   try {
-    const out = execSync('codex --version', { encoding: 'utf8', timeout: 5000 }).trim();
+    const out = execSync("codex --version", {
+      encoding: "utf8",
+      timeout: 5000,
+    }).trim();
     const match = out.match(/(\d+)\.(\d+)\.(\d+)/);
     _cachedVersion = match ? Number.parseInt(match[2], 10) : 0;
   } catch {
-    _cachedVersion = 0;
+    // Command builders should remain stable in CI even when the real Codex
+    // CLI is absent. Runtime preflight still reports/install-gates Codex
+    // separately; this fallback only selects the modern argv shape.
+    _cachedVersion = 117;
   }
   return _cachedVersion;
 }
@@ -43,19 +110,27 @@ export function gte(minMinor) {
  */
 export const FEATURES = {
   /** exec 서브커맨드 사용 가능 여부 (0.110+ 이전부터 존재) */
-  get execSubcommand() { return gte(110); },
+  get execSubcommand() {
+    return gte(110);
+  },
   /** --output-last-message 플래그 지원 여부 (0.117+) */
-  get outputLastMessage() { return gte(117); },
+  get outputLastMessage() {
+    return gte(117);
+  },
   /** --color <COLOR> 플래그 지원 여부 (exec와 동시 도입) */
-  get colorNever() { return gte(110); },
+  get colorNever() {
+    return gte(110);
+  },
   /** 플러그인 시스템 지원 여부 (향후 확장용) */
-  get pluginSystem() { return gte(120); },
+  get pluginSystem() {
+    return gte(120);
+  },
 };
 
 // ── Shell utilities ─────────────────────────────────────────────
 
 export function normalizePathForShell(value) {
-  return IS_WINDOWS ? String(value).replace(/\\/g, '/') : String(value);
+  return IS_WINDOWS ? String(value).replace(/\\/g, "/") : String(value);
 }
 
 export function shellQuote(value) {
@@ -71,47 +146,84 @@ export const CODEX_MCP_EXECUTION_EXIT_CODE = 1;
 
 /**
  * long-form 플래그 기반 명령 빌더.
+ *
+ * macOS 회귀 fix (codex v0.130.0 oh-my-codex hook lifecycle): 매우 긴
+ * argv-inline prompt 가 SessionStart Failed 를 유발하므로 default 가 prompt
+ * 를 stdin 으로 전달하는 패턴. 셸별 분기:
+ *   - Unix (bash/zsh): `codex exec ... < '/tmp/triflux-codex-prompt/prompt-*.txt'`
+ *   - Windows (pwsh7): `Get-Content -Raw 'path' | codex exec ...`
+ *     (pwsh7 의 `<` 는 reserved future syntax 라 호환성 보장 안 됨)
+ *
+ * 환경 변수 `TFX_CODEX_STDIN_PROMPT=0` 이면 legacy argv-inline 으로 회귀.
+ * 결정론 보장이 필요한 launcher path 는 호출 시 `stdinPrompt: false` 명시.
+ *
  * @param {string} prompt
  * @param {string|null} resultFile — null이면 --output-last-message 생략
- * @param {{ profile?: string, skipGitRepoCheck?: boolean, sandboxBypass?: boolean, cwd?: string, mcpServers?: string[] }} [opts]
+ * @param {{ profile?: string, skipGitRepoCheck?: boolean, sandboxBypass?: boolean, cwd?: string, mcpServers?: string[], stdinPrompt?: boolean }} [opts]
  * @returns {string} 실행할 셸 커맨드
  */
 export function buildExecCommand(prompt, resultFile = null, opts = {}) {
-  const { profile, skipGitRepoCheck = true, sandboxBypass = true, cwd, mcpServers } = opts;
+  const {
+    profile,
+    skipGitRepoCheck = true,
+    sandboxBypass = true,
+    mcpServers,
+    stdinPrompt,
+  } = opts;
 
-  const parts = ['codex'];
-  if (profile) parts.push('--profile', profile);
+  const parts = ["codex"];
+  if (profile) parts.push("--profile", profile);
 
   if (FEATURES.execSubcommand) {
-    parts.push('exec');
-    if (sandboxBypass) parts.push('--dangerously-bypass-approvals-and-sandbox');
-    if (skipGitRepoCheck) parts.push('--skip-git-repo-check');
+    parts.push("exec");
+    if (sandboxBypass) parts.push("--dangerously-bypass-approvals-and-sandbox");
+    if (skipGitRepoCheck) parts.push("--skip-git-repo-check");
     if (resultFile && FEATURES.outputLastMessage) {
-      parts.push('--output-last-message', resultFile);
+      parts.push("--output-last-message", resultFile);
     }
-    if (FEATURES.colorNever) parts.push('--color', 'never');
-    if (cwd) parts.push('--cwd', `'${escapePwshSingleQuoted(cwd)}'`);
+    if (FEATURES.colorNever) parts.push("--color", "never");
+    // NOTE: `codex exec`는 --cwd 플래그를 지원하지 않는다. Node spawn의 cwd
+    // 옵션으로 child process의 working directory를 제어한다 (conductor.mjs 참조).
+    // opts.cwd는 기록용으로만 받아두고 CLI command에는 반영하지 않는다.
+    // (이전 커밋에서 잘못 추가되어 shard 전체가 exit 2로 크래시한 회귀 #94 후속 이슈)
     if (Array.isArray(mcpServers)) {
       for (const server of mcpServers) {
-        parts.push('-c', `mcp_servers.${server}.enabled=true`);
+        parts.push("-c", `mcp_servers.${server}.enabled=true`);
       }
     }
   } else {
-    parts.push('--dangerously-bypass-approvals-and-sandbox');
-    if (skipGitRepoCheck) parts.push('--skip-git-repo-check');
+    parts.push("--dangerously-bypass-approvals-and-sandbox");
+    if (skipGitRepoCheck) parts.push("--skip-git-repo-check");
+  }
+
+  const useStdin = resolveStdinPromptMode(stdinPrompt);
+  const hasPrompt = typeof prompt === "string" && prompt.length > 0;
+  if (useStdin && hasPrompt) {
+    const promptFile = writePromptToTmpFile(prompt);
+    if (IS_WINDOWS) {
+      // pwsh7 호환: `<` redirect 는 reserved future syntax 라 동작 불안정.
+      // Get-Content -Raw stdin pipe 로 prompt 를 codex stdin 에 주입.
+      return `Get-Content -Raw '${escapePwshSingleQuoted(promptFile)}' | ${parts.join(" ")}`;
+    }
+    return `${parts.join(" ")} < ${shellQuote(promptFile)}`;
   }
 
   parts.push(JSON.stringify(prompt));
-  return parts.join(' ');
+  return parts.join(" ");
+}
+
+function resolveStdinPromptMode(explicit) {
+  if (typeof explicit === "boolean") return explicit;
+  // Default ON to fix macOS codex hook regression. Opt out via env=0.
+  const env = process.env.TFX_CODEX_STDIN_PROMPT;
+  if (env === "0" || env === "false") return false;
+  return true;
 }
 
 // ── Sleep ───────────────────────────────────────────────────────
 
 export function sleep(ms) {
-  return new Promise((resolve) => {
-    const timer = setTimeout(resolve, ms);
-    timer.unref?.();
-  });
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
 // ── Result factory ──────────────────────────────────────────────
@@ -119,80 +231,138 @@ export function sleep(ms) {
 export function createResult(ok, extra = {}) {
   return {
     ok,
-    output: '',
-    stderr: '',
+    output: "",
+    stderr: "",
     exitCode: null,
     duration: 0,
     retried: false,
     fellBack: false,
-    failureMode: ok ? null : 'crash',
+    failureMode: ok ? null : "crash",
     ...extra,
   };
 }
 
 export function appendWarnings(stderr, warnings = []) {
-  const text = warnings.map((item) => `[preflight] ${item}`).join('\n');
-  return [stderr, text].filter(Boolean).join('\n');
+  const text = warnings.map((item) => `[preflight] ${item}`).join("\n");
+  return [stderr, text].filter(Boolean).join("\n");
 }
 
-// ── Circuit breaker factory ─────────────────────────────────────
-
-export function createCircuitBreaker(opts = {}) {
-  const state = {
-    failures: [],
-    maxFailures: opts.maxFailures ?? 3,
-    windowMs: opts.windowMs ?? 10 * 60_000,
-    openedAt: 0,
-    trialInFlight: false,
-  };
+// ── Broker-integrated execution ─────────────────────────────────
 
-  function pruneFailures(now = Date.now()) {
-    state.failures = state.failures.filter((stamp) => now - stamp < state.windowMs);
+/**
+ * Shared execute() logic that uses account-broker for per-account circuit
+ * breaking instead of a global breaker.
+ *
+ * @param {object} params
+ * @param {string} params.provider — 'codex' | 'gemini'
+ * @param {(prompt: string, workdir: string, preflight: object, attempt: object) => Promise<object>} params.runFn
+ * @param {(opts: object) => Promise<object>} params.preflightFn
+ * @param {(opts: object, preflight: object) => object[]} params.buildAttemptsFn
+ * @param {object} params.opts — caller-supplied execute options
+ * @returns {Promise<object>} createResult-shaped result
+ */
+export async function executeWithCircuitBroker({
+  provider,
+  runFn,
+  preflightFn,
+  buildAttemptsFn,
+  opts = {},
+}) {
+  // late-import to avoid circular dependency at module load time
+  const brokerMod = await import("./account-broker.mjs");
+  const { withRetry } = await import("./workers/worker-utils.mjs");
+
+  // access broker as live binding property (not destructured) so reloadBroker() propagates
+  const hasBroker = brokerMod.broker != null;
+  // Empty broker (TFX_DISABLE_ACCOUNT_BROKER=1 or zero accounts) must not
+  // gate execution. lease() is null because there are no accounts to lease,
+  // not because all accounts are busy/cooldown/circuit. Treat it like
+  // hasBroker=false and fall through to the default auth path.
+  const brokerDisabled = hasBroker && brokerMod.broker.isDisabled === true;
+  const effectiveBroker = hasBroker && !brokerDisabled;
+  const lease = effectiveBroker ? brokerMod.broker.lease({ provider }) : null;
+  if (effectiveBroker && !lease) {
+    return createResult(false, { fellBack: true, failureMode: "circuit_open" });
   }
 
-  function reset() {
-    state.failures = [];
-    state.openedAt = 0;
-    state.trialInFlight = false;
+  const preflight = await preflightFn(opts);
+  if (!preflight.ok) {
+    if (lease) brokerMod.broker.release(lease.id, { ok: false });
+    return createResult(false, {
+      stderr: appendWarnings("", preflight.warnings),
+      fellBack: opts.fallbackToClaude !== false,
+      failureMode: "crash",
+    });
   }
 
-  function recordFailure(isHalfOpen, now = Date.now()) {
-    pruneFailures(now);
-    state.failures = [...state.failures, now];
-    state.trialInFlight = false;
-    if (isHalfOpen || state.failures.length >= state.maxFailures) {
-      state.openedAt = now;
-    }
-  }
+  const attempts = buildAttemptsFn(opts, preflight);
+  let attemptIndex = 0;
+  let lastResult = createResult(false);
 
-  function getState(now = Date.now()) {
-    pruneFailures(now);
-    const withinWindow = state.openedAt && now - state.openedAt < state.windowMs;
-    const current = withinWindow ? 'open' : (state.openedAt ? 'half-open' : 'closed');
-    return {
-      state: current,
-      failures: [...state.failures],
-      maxFailures: state.maxFailures,
-      windowMs: state.windowMs,
-      openedAt: state.openedAt || null,
-      trialInFlight: state.trialInFlight,
-    };
+  try {
+    lastResult = await withRetry(
+      async () => {
+        // PRD A1: lease 메타데이터를 runFn 에 전달해서 adapter 가 실제 spawn 시
+        // 해당 account 의 authFile/env/profile 을 적용할 수 있게 한다. lease=null
+        // 이면 adapter 는 default 경로 (단일 ~/.codex/auth.json) 로 동작한다.
+        const result = await runFn(
+          opts.prompt || "",
+          opts.workdir || process.cwd(),
+          preflight,
+          attempts[attemptIndex],
+          lease,
+        );
+        const current = {
+          ...result,
+          stderr: appendWarnings(result.stderr, preflight.warnings),
+          retried: attemptIndex > 0,
+        };
+        const canRetry = !current.ok && attemptIndex < attempts.length - 1;
+        attemptIndex += 1;
+        if (!canRetry) return current;
+        const error = new Error("retry");
+        error.retryable = true;
+        error.result = current;
+        throw error;
+      },
+      {
+        maxAttempts: attempts.length,
+        baseDelayMs: 250,
+        maxDelayMs: 750,
+        shouldRetry: (error) => error?.retryable === true,
+      },
+    );
+  } catch (error) {
+    lastResult =
+      error?.result ||
+      createResult(false, { stderr: String(error?.message || error) });
   }
 
-  function canExecute() {
-    const circuit = getState();
-    if (circuit.state === 'open') return { allowed: false, halfOpen: false };
-    if (circuit.state === 'half-open' && state.trialInFlight) return { allowed: false, halfOpen: true };
-    const halfOpen = circuit.state === 'half-open';
-    if (halfOpen) state.trialInFlight = true;
-    return { allowed: true, halfOpen };
+  if (lastResult.ok) {
+    if (lease) brokerMod.broker.release(lease.id, { ok: true });
+    return lastResult;
   }
 
-  function clearTrial() {
-    state.trialInFlight = false;
+  if (lease) {
+    if (lastResult.failureMode === "rate_limited") {
+      const text = `${lastResult.output || ""}\n${lastResult.stderr || ""}`;
+      const coolMs = parseRetryAfterMs(text, provider);
+      brokerMod.broker.markRateLimited(lease.id, coolMs);
+      brokerMod.broker.emit("cooldown", {
+        id: lease.id,
+        provider,
+        coolMs,
+        reason: "quota_exhausted",
+      });
+    } else {
+      brokerMod.broker.release(lease.id, { ok: false });
+    }
   }
-
-  return { getState, recordFailure, reset, canExecute, clearTrial };
+  return {
+    ...lastResult,
+    retried: attempts.length > 1,
+    fellBack: opts.fallbackToClaude !== false,
+  };
 }
 
 // ── Process termination ─────────────────────────────────────────
@@ -200,9 +370,14 @@ export function createCircuitBreaker(opts = {}) {
 export async function terminateChild(pid, opts = {}) {
   if (!pid) return;
   const graceMs = opts.graceMs ?? 5000;
-  killProcess(pid, { signal: 'SIGTERM', tree: true, timeout: graceMs });
+  killProcess(pid, { signal: "SIGTERM", tree: true, timeout: graceMs });
   await sleep(graceMs);
-  killProcess(pid, { signal: 'SIGKILL', tree: true, force: true, timeout: graceMs });
+  killProcess(pid, {
+    signal: "SIGKILL",
+    tree: true,
+    force: true,
+    timeout: graceMs,
+  });
 }
 
 // ── Process execution with stall detection ──────────────────────
@@ -222,29 +397,61 @@ export async function terminateChild(pid, opts = {}) {
  */
 export async function runProcess(command, workdir, timeout, opts = {}) {
   const startedAt = Date.now();
-  const inferStallMode = opts.inferStallMode || (() => 'timeout');
+  const inferStallMode = opts.inferStallMode || (() => "timeout");
   const stallCheckIntervalMs = opts.stallCheckIntervalMs ?? 10_000;
   const stallThresholdMs = opts.stallThresholdMs ?? 30_000;
   const resultFile = opts.resultFile || null;
 
-  let stdout = '';
-  let stderr = '';
+  let stdout = "";
+  let stderr = "";
   let exitCode = null;
   let failureMode = null;
   let child;
 
   try {
-    child = spawn(command, { cwd: workdir, shell: true, windowsHide: true });
+    // PRD A1: opts.spawnEnv 가 있으면 그 env 로 spawn (lease 의 authFile/env 적용).
+    // undefined 면 spawn 의 default 동작 (부모 process env inherit) 유지.
+    child = spawn(command, {
+      cwd: workdir,
+      shell: true,
+      windowsHide: true,
+      env: opts.spawnEnv,
+    });
   } catch (error) {
-    return createResult(false, { stderr: String(error?.message || error), duration: Date.now() - startedAt });
+    return createResult(false, {
+      stderr: String(error?.message || error),
+      duration: Date.now() - startedAt,
+    });
   }
 
+  const resultFileSignature = () => {
+    if (!resultFile) return "";
+    try {
+      const info = statSync(resultFile);
+      return `${info.size}:${info.mtimeMs}`;
+    } catch {
+      return "";
+    }
+  };
+
   let lastBytes = 0;
+  let lastResultFileSignature = resultFileSignature();
   let lastChange = Date.now();
-  const touch = () => { lastChange = Date.now(); };
-  child.stdout?.on('data', (chunk) => { stdout += String(chunk); touch(); });
-  child.stderr?.on('data', (chunk) => { stderr += String(chunk); touch(); });
-  child.on('error', (error) => { stderr += String(error?.message || error); failureMode ||= 'crash'; });
+  const touch = () => {
+    lastChange = Date.now();
+  };
+  child.stdout?.on("data", (chunk) => {
+    stdout += String(chunk);
+    touch();
+  });
+  child.stderr?.on("data", (chunk) => {
+    stderr += String(chunk);
+    touch();
+  });
+  child.on("error", (error) => {
+    stderr += String(error?.message || error);
+    failureMode ||= "crash";
+  });
 
   const stopFor = async (mode) => {
     if (failureMode) return;
@@ -252,23 +459,40 @@ export async function runProcess(command, workdir, timeout, opts = {}) {
     await terminateChild(child.pid);
   };
 
-  const timeoutTimer = setTimeout(() => { void stopFor('timeout'); }, timeout);
+  const timeoutTimer = setTimeout(() => {
+    void stopFor("timeout");
+  }, timeout);
   const stallTimer = setInterval(() => {
     const size = Buffer.byteLength(stdout) + Buffer.byteLength(stderr);
-    if (size !== lastBytes) {
+    const currentResultFileSignature = resultFileSignature();
+    if (
+      size !== lastBytes ||
+      currentResultFileSignature !== lastResultFileSignature
+    ) {
       lastBytes = size;
+      lastResultFileSignature = currentResultFileSignature;
+      touch();
       return;
     }
-    if (Date.now() - lastChange >= stallThresholdMs) void stopFor(inferStallMode(stdout, stderr));
+    if (Date.now() - lastChange >= stallThresholdMs)
+      void stopFor(inferStallMode(stdout, stderr));
   }, stallCheckIntervalMs);
   timeoutTimer.unref?.();
   stallTimer.unref?.();
 
-  await new Promise((resolve) => child.on('close', (code) => { exitCode = code; resolve(); }));
+  await new Promise((resolve) =>
+    child.on("close", (code) => {
+      exitCode = code;
+      resolve();
+    }),
+  );
   clearTimeout(timeoutTimer);
   clearInterval(stallTimer);
 
-  const fileOutput = resultFile && existsSync(resultFile) ? readFileSync(resultFile, 'utf8') : '';
+  const fileOutput =
+    resultFile && existsSync(resultFile)
+      ? readFileSync(resultFile, "utf8")
+      : "";
   const output = fileOutput || stdout;
   const ok = failureMode == null && exitCode === 0;
   return createResult(ok, {
@@ -276,6 +500,6 @@ export async function runProcess(command, workdir, timeout, opts = {}) {
     stderr,
     exitCode,
     duration: Date.now() - startedAt,
-    failureMode: ok ? null : (failureMode || 'crash'),
+    failureMode: ok ? null : failureMode || "crash",
   });
 }