@jsonstudio/rcc 0.89.1803 → 0.89.1959

package/dist/server/runtime/http-server/request-executor.js

@@ -4,12 +4,69 @@ import { attachProviderRuntimeMetadata } from '../../../providers/core/runtime/p
 import { extractAnthropicToolAliasMap } from './anthropic-tool-alias.js';
 import { enhanceProviderRequestId } from '../../utils/request-id-manager.js';
 import { buildRequestMetadata, cloneClientHeaders, decorateMetadataForAttempt, ensureClientHeadersOnPayload, resolveClientRequestId } from './executor-metadata.js';
-import { describeRetryReason, isNetworkTransportError, shouldRetryProviderError, waitBeforeRetry } from './executor-provider.js';
+import { describeRetryReason, shouldRetryProviderError, waitBeforeRetry } from './executor-provider.js';
 import { convertProviderResponse as bridgeConvertProviderResponse, createSnapshotRecorder as bridgeCreateSnapshotRecorder, } from '../../../modules/llmswitch/bridge.js';
+import { getClockClientRegistry, injectClockClientPrompt } from './clock-client-registry.js';
 import { ensureHubPipeline, runHubPipeline } from './executor-pipeline.js';
 import { buildInfo } from '../../../build-info.js';
 const DEFAULT_MAX_PROVIDER_ATTEMPTS = 6;
 const DEFAULT_ANTIGRAVITY_MAX_PROVIDER_ATTEMPTS = 20;
+const RETRYABLE_SSE_ERROR_CODE_HINTS = [
+    'internal_network_failure',
+    'network_error',
+    'api_connection_error',
+    'service_unavailable',
+    'internal_server_error',
+    'overloaded_error',
+    'rate_limit_error',
+    'request_timeout',
+    'timeout'
+];
+const RETRYABLE_SSE_MESSAGE_HINTS = [
+    'internal network failure',
+    'network failure',
+    'network error',
+    'temporarily unavailable',
+    'temporarily unreachable',
+    'upstream disconnected',
+    'connection reset',
+    'connection closed',
+    'timed out',
+    'timeout'
+];
+function firstNonEmptyString(candidates) {
+    for (const candidate of candidates) {
+        if (typeof candidate !== 'string') {
+            continue;
+        }
+        const trimmed = candidate.trim();
+        if (trimmed) {
+            return trimmed;
+        }
+    }
+    return undefined;
+}
+function firstFiniteNumber(candidates) {
+    for (const candidate of candidates) {
+        if (typeof candidate === 'number' && Number.isFinite(candidate)) {
+            return candidate;
+        }
+    }
+    return undefined;
+}
+function isRetryableSseWrapperError(message, errorCode, status) {
+    if (typeof status === 'number' && Number.isFinite(status)) {
+        if (status === 408 || status === 425 || status === 429 || status >= 500) {
+            return true;
+        }
+    }
+    const normalizedCode = typeof errorCode === 'string' ? errorCode.trim().toLowerCase() : '';
+    if (normalizedCode && RETRYABLE_SSE_ERROR_CODE_HINTS.some((hint) => normalizedCode.includes(hint))) {
+        return true;
+    }
+    const loweredMessage = message.toLowerCase();
+    return RETRYABLE_SSE_MESSAGE_HINTS.some((hint) => loweredMessage.includes(hint));
+}
 function resolveBoolFromEnv(value, fallback) {
     if (!value) {
         return fallback;
@@ -143,6 +200,48 @@ function injectAntigravityRetrySignal(metadata, signal) {
         ...(signal.avoidAllOnRetry === true ? { antigravityAvoidAllOnRetry: true } : {})
     };
 }
+function normalizeSessionToken(value) {
+    if (typeof value !== 'string') {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed || undefined;
+}
+function inferClockClientTypeFromMetadata(metadata) {
+    const direct = normalizeSessionToken(metadata.clockClientType) ?? normalizeSessionToken(metadata.clientType);
+    if (direct) {
+        return direct;
+    }
+    const userAgent = normalizeSessionToken(metadata.userAgent)?.toLowerCase() ?? '';
+    if (userAgent.includes('codex')) {
+        return 'codex';
+    }
+    if (userAgent.includes('claude')) {
+        return 'claude';
+    }
+    return undefined;
+}
+function bindClockConversationSession(metadata) {
+    const conversationSessionId = normalizeSessionToken(metadata.sessionId);
+    if (!conversationSessionId) {
+        return;
+    }
+    const tmuxSessionId = normalizeSessionToken(metadata.tmuxSessionId);
+    const daemonId = normalizeSessionToken(metadata.clockDaemonId)
+        ?? normalizeSessionToken(metadata.clockClientDaemonId);
+    const clientType = inferClockClientTypeFromMetadata(metadata);
+    try {
+        getClockClientRegistry().bindConversationSession({
+            conversationSessionId,
+            ...(tmuxSessionId ? { tmuxSessionId } : {}),
+            ...(daemonId ? { daemonId } : {}),
+            ...(clientType ? { clientType } : {})
+        });
+    }
+    catch {
+        // best-effort only
+    }
+}
 export class HubRequestExecutor {
     deps;
     constructor(deps) {
@@ -161,6 +260,7 @@ export class HubRequestExecutor {
         try {
             const hubPipeline = ensureHubPipeline(this.deps.getHubPipeline);
             const initialMetadata = buildRequestMetadata(input);
+            bindClockConversationSession(initialMetadata);
             const inboundClientHeaders = cloneClientHeaders(initialMetadata?.clientHeaders);
             const providerRequestId = input.requestId;
             const clientRequestId = resolveClientRequestId(initialMetadata, providerRequestId);
@@ -217,7 +317,26 @@ export class HubRequestExecutor {
                     stream: metadataForAttempt.stream,
                     attempt
                 });
-                const pipelineResult = await runHubPipeline(hubPipeline, input, metadataForAttempt);
+                let pipelineResult;
+                try {
+                    pipelineResult = await runHubPipeline(hubPipeline, input, metadataForAttempt);
+                }
+                catch (pipelineError) {
+                    const pipelineErrorCode = typeof pipelineError.code === 'string'
+                        ? String(pipelineError.code).trim()
+                        : '';
+                    const pipelineErrorMessage = pipelineError instanceof Error
+                        ? pipelineError.message
+                        : String(pipelineError ?? 'Unknown error');
+                    const isPoolExhaustedError = pipelineErrorCode === 'PROVIDER_NOT_AVAILABLE' ||
+                        pipelineErrorCode === 'ERR_NO_PROVIDER_TARGET' ||
+                        /all providers unavailable/i.test(pipelineErrorMessage) ||
+                        /virtual router did not produce a provider target/i.test(pipelineErrorMessage);
+                    if (lastError && isPoolExhaustedError) {
+                        throw lastError;
+                    }
+                    throw pipelineError;
+                }
                 const pipelineMetadata = pipelineResult.metadata ?? {};
                 const mergedMetadata = { ...metadataForAttempt, ...pipelineMetadata };
                 const mergedClientHeaders = cloneClientHeaders(mergedMetadata?.clientHeaders) || clientHeadersForAttempt;
@@ -251,6 +370,38 @@ export class HubRequestExecutor {
                 }
                 const runtimeKey = target.runtimeKey || this.deps.runtimeManager.resolveRuntimeKey(target.providerKey);
                 if (!runtimeKey) {
+                    const runtimeResolveError = Object.assign(new Error(`Runtime for provider ${target.providerKey} not initialized`), {
+                        code: 'ERR_RUNTIME_NOT_FOUND',
+                        requestId: input.requestId,
+                        retryable: true
+                    });
+                    try {
+                        const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
+                        emitProviderError({
+                            error: runtimeResolveError,
+                            stage: 'provider.runtime.resolve',
+                            runtime: {
+                                requestId: input.requestId,
+                                providerKey: target.providerKey,
+                                providerId: target.providerKey.split('.')[0],
+                                providerType: String(target.providerType || 'unknown'),
+                                providerProtocol: String(target.outboundProfile || ''),
+                                routeName: pipelineResult.routingDecision?.routeName,
+                                pipelineId: target.providerKey,
+                                target
+                            },
+                            dependencies: this.deps.getModuleDependencies(),
+                            recoverable: false,
+                            affectsHealth: true,
+                            details: {
+                                reason: 'runtime_not_initialized',
+                                providerKey: target.providerKey
+                            }
+                        });
+                    }
+                    catch {
+                        // best-effort
+                    }
                     throw Object.assign(new Error(`Runtime for provider ${target.providerKey} not initialized`), {
                         code: 'ERR_RUNTIME_NOT_FOUND',
                         requestId: input.requestId
@@ -258,6 +409,40 @@ export class HubRequestExecutor {
                 }
                 const handle = this.deps.runtimeManager.getHandleByRuntimeKey(runtimeKey);
                 if (!handle) {
+                    const runtimeMissingError = Object.assign(new Error(`Provider runtime ${runtimeKey} not found`), {
+                        code: 'ERR_PROVIDER_NOT_FOUND',
+                        requestId: input.requestId,
+                        retryable: true
+                    });
+                    try {
+                        const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
+                        emitProviderError({
+                            error: runtimeMissingError,
+                            stage: 'provider.runtime.resolve',
+                            runtime: {
+                                requestId: input.requestId,
+                                providerKey: target.providerKey,
+                                providerId: target.providerKey.split('.')[0],
+                                providerType: String(target.providerType || 'unknown'),
+                                providerProtocol: String(target.outboundProfile || ''),
+                                routeName: pipelineResult.routingDecision?.routeName,
+                                pipelineId: target.providerKey,
+                                runtimeKey,
+                                target
+                            },
+                            dependencies: this.deps.getModuleDependencies(),
+                            recoverable: false,
+                            affectsHealth: true,
+                            details: {
+                                reason: 'runtime_handle_missing',
+                                providerKey: target.providerKey,
+                                runtimeKey
+                            }
+                        });
+                    }
+                    catch {
+                        // best-effort
+                    }
                     throw Object.assign(new Error(`Provider runtime ${runtimeKey} not found`), {
                         code: 'ERR_PROVIDER_NOT_FOUND',
                         requestId: input.requestId
@@ -364,22 +549,57 @@ export class HubRequestExecutor {
                         response: normalized,
                         pipelineMetadata: mergedMetadata
                     });
-                    // For Antigravity/Gemini, we may receive an error-shaped response (status=429/400) instead of a thrown error,
-                    // so llmswitch-core servertool can attempt thoughtSignature bootstrap + replay.
-                    // If it is still an error after conversion, treat it as a provider failure and continue the retry loop.
-                    if (typeof converted.status === 'number' &&
-                        converted.status >= 400 &&
+                    // Treat upstream 429 as provider failure across protocols to avoid
+                    // silently returning success and to let Virtual Router failover to other candidates.
+                    // Keep existing Gemini compatibility behavior for 400/4xx thoughtSignature-like failures.
+                    const convertedStatus = typeof converted.status === 'number' ? converted.status : undefined;
+                    const isGlobalRetryable429 = convertedStatus === 429;
+                    const isGeminiCompatFailure = typeof convertedStatus === 'number' &&
+                        convertedStatus >= 400 &&
                         (isAntigravityProviderKey(target.providerKey) ||
                             (typeof target.providerKey === 'string' && target.providerKey.startsWith('gemini-cli.'))) &&
-                        providerProtocol === 'gemini-chat') {
+                        providerProtocol === 'gemini-chat';
+                    if (isGlobalRetryable429 || isGeminiCompatFailure) {
                         const bodyForError = converted.body && typeof converted.body === 'object' ? converted.body : undefined;
                         const errMsg = bodyForError && bodyForError.error && typeof bodyForError.error === 'object'
                             ? String(bodyForError.error.message || bodyForError.error || '')
                             : '';
-                        const errorToThrow = new Error(errMsg && errMsg.trim().length ? errMsg : `HTTP ${converted.status}`);
-                        errorToThrow.statusCode = converted.status;
-                        errorToThrow.status = converted.status;
+                        const statusCode = typeof convertedStatus === 'number' ? convertedStatus : 500;
+                        const errorToThrow = new Error(errMsg && errMsg.trim().length ? errMsg : `HTTP ${statusCode}`);
+                        errorToThrow.statusCode = statusCode;
+                        errorToThrow.status = statusCode;
                         errorToThrow.response = { data: bodyForError };
+                        try {
+                            const { emitProviderError } = await import('../../../providers/core/utils/provider-error-reporter.js');
+                            emitProviderError({
+                                error: errorToThrow,
+                                stage: 'provider.http',
+                                runtime: {
+                                    requestId: input.requestId,
+                                    providerKey: target.providerKey,
+                                    providerId: handle.providerId,
+                                    providerType: handle.providerType,
+                                    providerFamily: handle.providerFamily,
+                                    providerProtocol,
+                                    routeName: pipelineResult.routingDecision?.routeName,
+                                    pipelineId: target.providerKey,
+                                    target,
+                                    runtimeKey
+                                },
+                                dependencies: this.deps.getModuleDependencies(),
+                                statusCode,
+                                recoverable: statusCode === 429,
+                                affectsHealth: true,
+                                details: {
+                                    source: 'converted_response_status',
+                                    convertedStatus: statusCode,
+                                    wrappedErrorResponse: true
+                                }
+                            });
+                        }
+                        catch {
+                            // best-effort; never block retry/failover path
+                        }
                         throw errorToThrow;
                     }
                     const usage = this.extractUsageFromResult(converted, mergedMetadata);
@@ -434,11 +654,9 @@ export class HubRequestExecutor {
                     recordAttempt({ error: true });
                     const singleProviderPool = Boolean(initialRoutePool && initialRoutePool.length === 1 && initialRoutePool[0] === target.providerKey);
                     if (singleProviderPool) {
-                        if (isNetworkTransportError(error)) {
-                            await waitBeforeRetry(error);
-                        }
+                        await waitBeforeRetry(error);
                     }
-                    else if (target.providerKey) {
+                    if (!singleProviderPool && target.providerKey) {
                         const is429 = status === 429;
                         if (isAntigravityProviderKey(target.providerKey) && (isVerify || is429)) {
                             // For Antigravity 403 verify / 429 states:
@@ -515,12 +733,21 @@ export class HubRequestExecutor {
         if (body && typeof body === 'object') {
             const wrapperError = this.extractSseWrapperError(body);
             if (wrapperError) {
-                const error = new Error(`[RequestExecutor] Upstream SSE terminated: ${wrapperError}`);
+                const codeSuffix = wrapperError.errorCode ? ` [${wrapperError.errorCode}]` : '';
+                const error = new Error(`Upstream SSE error event${codeSuffix}: ${wrapperError.message}`);
                 error.code = 'SSE_DECODE_ERROR';
+                if (wrapperError.errorCode) {
+                    error.upstreamCode = wrapperError.errorCode;
+                }
+                error.retryable = wrapperError.retryable;
+                if (wrapperError.retryable) {
+                    error.status = 503;
+                    error.statusCode = 503;
+                }
                 throw error;
             }
         }
-        if (options.processMode === 'passthrough') {
+        if (options.processMode === 'passthrough' && !options.wantsStream) {
             return options.response;
         }
         const entry = (options.entryEndpoint || '').toLowerCase();
@@ -550,6 +777,12 @@ export class HubRequestExecutor {
             const baseContext = {
                 ...(metadataBag ?? {})
             };
+            if (baseContext.capturedChatRequest === undefined &&
+                options.originalRequest &&
+                typeof options.originalRequest === 'object' &&
+                !Array.isArray(options.originalRequest)) {
+                baseContext.capturedChatRequest = options.originalRequest;
+            }
             if (typeof metadataBag?.routeName === 'string') {
                 baseContext.routeId = metadataBag.routeName;
             }
@@ -636,6 +869,7 @@ export class HubRequestExecutor {
                     delete nestedMetadata.clientHeaders;
                     delete nestedMetadata.clientRequestId;
                 }
+                bindClockConversationSession(nestedMetadata);
                 const nestedInput = {
                     entryEndpoint: nestedEntry,
                     method: 'POST',
@@ -645,6 +879,26 @@ export class HubRequestExecutor {
                     body: reenterOpts.body,
                     metadata: nestedMetadata
                 };
+                try {
+                    const requestBody = reenterOpts.body;
+                    const messages = Array.isArray(requestBody.messages) ? requestBody.messages : [];
+                    const lastUser = [...messages]
+                        .reverse()
+                        .find((entry) => entry && typeof entry === 'object' && entry.role === 'user');
+                    const text = typeof lastUser?.content === 'string' ? String(lastUser.content) : '';
+                    if (text.includes('<**clock:{') && text.includes('}**>')) {
+                        await injectClockClientPrompt({
+                            tmuxSessionId: typeof nestedMetadata.tmuxSessionId === 'string' ? nestedMetadata.tmuxSessionId : undefined,
+                            sessionId: typeof nestedMetadata.sessionId === 'string' ? nestedMetadata.sessionId : undefined,
+                            text,
+                            requestId: reenterOpts.requestId,
+                            source: 'servertool.reenter'
+                        });
+                    }
+                }
+                catch {
+                    // best-effort only
+                }
                 const nestedResult = await this.execute(nestedInput);
                 const nestedBody = nestedResult.body && typeof nestedResult.body === 'object'
                     ? nestedResult.body
@@ -668,23 +922,10 @@ export class HubRequestExecutor {
                     body: { __sse_responses: converted.__sse_responses }
                 };
             }
-            const merged = {
+            return {
                 ...options.response,
                 body: converted.body ?? body
             };
-            // Special case: some Antigravity/Gemini upstream failures are intentionally passed through as
-            // an error-shaped response (status=429/400) so llmswitch-core servertool can recover via followup.
-            // If recovery succeeded, the final client payload will NOT contain an error envelope.
-            if (typeof options.response.status === 'number' && options.response.status >= 400) {
-                const out = merged.body;
-                if (out && typeof out === 'object' && !Array.isArray(out)) {
-                    const hasError = Object.prototype.hasOwnProperty.call(out, 'error');
-                    if (!hasError) {
-                        merged.status = 200;
-                    }
-                }
-            }
-            return merged;
         }
         catch (error) {
             const err = error;
@@ -714,10 +955,11 @@ export class HubRequestExecutor {
         if (!record || typeof record !== 'object' || depth < 0) {
             return undefined;
         }
-        const mode = record.mode;
-        const errVal = record.error;
-        if (mode === 'sse' && typeof errVal === 'string' && errVal.trim()) {
-            return errVal.trim();
+        if (record.mode === 'sse') {
+            const normalized = this.normalizeSseWrapperErrorValue(record.error, depth);
+            if (normalized) {
+                return normalized;
+            }
         }
         const nestedKeys = ['body', 'data', 'payload', 'response'];
         for (const key of nestedKeys) {
@@ -732,6 +974,89 @@ export class HubRequestExecutor {
         }
         return undefined;
     }
+    normalizeSseWrapperErrorValue(value, depth) {
+        if (value === undefined || value === null || depth < 0) {
+            return undefined;
+        }
+        if (typeof value === 'string') {
+            const trimmed = value.trim();
+            if (!trimmed) {
+                return undefined;
+            }
+            if (depth > 0 && (trimmed.startsWith('{') || trimmed.startsWith('['))) {
+                try {
+                    const parsed = JSON.parse(trimmed);
+                    const parsedInfo = this.normalizeSseWrapperErrorValue(parsed, depth - 1);
+                    if (parsedInfo) {
+                        return parsedInfo;
+                    }
+                }
+                catch {
+                    // fallback to raw string
+                }
+            }
+            return {
+                message: trimmed,
+                retryable: isRetryableSseWrapperError(trimmed)
+            };
+        }
+        if (typeof value !== 'object' || Array.isArray(value)) {
+            return undefined;
+        }
+        const record = value;
+        const directMessage = firstNonEmptyString([
+            record.message,
+            record.error_message,
+            record.errorMessage
+        ]);
+        const directCode = firstNonEmptyString([
+            record.code,
+            record.error_code,
+            record.errorCode,
+            record.type
+        ]);
+        const directStatus = firstFiniteNumber([
+            record.status,
+            record.statusCode,
+            record.status_code,
+            record.http_status
+        ]);
+        if (depth > 0) {
+            for (const key of ['error', 'data', 'payload', 'details', 'body', 'response']) {
+                const nestedInfo = this.normalizeSseWrapperErrorValue(record[key], depth - 1);
+                if (nestedInfo) {
+                    const mergedCode = nestedInfo.errorCode ?? directCode;
+                    const retryable = nestedInfo.retryable || isRetryableSseWrapperError(nestedInfo.message, mergedCode, directStatus);
+                    return {
+                        message: nestedInfo.message,
+                        ...(mergedCode ? { errorCode: mergedCode } : {}),
+                        retryable
+                    };
+                }
+            }
+        }
+        if (directMessage) {
+            return {
+                message: directMessage,
+                ...(directCode ? { errorCode: directCode } : {}),
+                retryable: isRetryableSseWrapperError(directMessage, directCode, directStatus)
+            };
+        }
+        try {
+            const serialized = JSON.stringify(record);
+            if (serialized && serialized !== '{}') {
+                return {
+                    message: serialized,
+                    ...(directCode ? { errorCode: directCode } : {}),
+                    retryable: isRetryableSseWrapperError(serialized, directCode, directStatus)
+                };
+            }
+        }
+        catch {
+            // ignore stringify failures
+        }
+        return undefined;
+    }
     extractClientModelId(metadata, originalRequest) {
         const candidates = [
             metadata.clientModelId,