@cc-monitoring/proxy-core 0.3.1 → 0.4.0

package/dist/server/proxy.js

@@ -1,16 +1,17 @@
 import { Readable, finished } from 'stream';
 import { convertClaudeToOpenAIRequest, convertOpenAIToClaudeResponse, createStreamTransformer, buildOpenAIHeaders } from '../utils/openai-converter.js';
 import { loadConfig } from '../config/index.js';
-import { saveRequestDB, extractTokenUsage, flushEventBatch, deleteResponseEvents, deleteRequestRow, updateRequestTokens, EVENT_BATCH_SIZE } from '../utils/request-writer.js';
-import { cleanOldRequestsDB } from '../utils/request-writer.js';
+import { createRequestRecorder } from '../utils/request-writer.js';
+import { cleanOldRequestsDB, interruptStaleRequestsOnStartup } from '../utils/request-writer.js';
 import { pushNewRequest } from './data-push.js';
 import { initDatabase, closeDatabase, getDatabase } from '../utils/db.js';
+import { recoverCrashedTasks } from '../utils/learn-tasks-db.js';
+import { getActiveProvider } from '../utils/providers-db.js';
 import { aggregateStats } from '../utils/stats-aggregator.js';
 import { mcpManager } from '../utils/mcp-manager.js';
 import { getMachineIdSync, preloadMachineId } from '../utils/machine-id.js';
-// Cached prepared statement for active provider lookup (avoids per-request allocation)
-let activeProviderStmt = null;
-let cachedDbInstance = null;
+import { createEventDecoder } from '../utils/sse-protocol.js';
+import { createClientAbortSignal, createTimeoutSignal } from '../utils/abort-signals.js';
 /**
  * Resolves the proxy target per request:
  * - If a provider with is_default=1 exists in DB, use its base_url + api_key + tier models (partial replacement)
@@ -36,15 +37,7 @@ export function resolveProxyTarget(proxyConfig) {
         baseUrl = configUrl.origin;
         targetPath = configUrl.pathname.replace(/\/$/, '');
         const db = getDatabase();
-        // Invalidate cached stmt if DB instance changed (happens in tests with fresh DB per test)
-        if (db !== cachedDbInstance) {
-            activeProviderStmt = null;
-            cachedDbInstance = db;
-        }
-        if (!activeProviderStmt) {
-            activeProviderStmt = db.prepare('SELECT base_url, api_key, model, haiku_model, sonnet_model, opus_model, is_openai_compatible, auth_style FROM providers WHERE is_default = 1 LIMIT 1');
-        }
-        const active = activeProviderStmt.get();
+        const active = getActiveProvider(db);
         if (active) {
             // Replace base_url only if provided
             if (active.base_url && active.base_url.trim() !== '') {
@@ -163,58 +156,6 @@ export function getErrorStatusCode(error) {
     }
     return 500;
 }
-/**
- * Push a parsed SSE event into batch/tail buffers.
- * Flushes batch to DB when it reaches EVENT_BATCH_SIZE (10).
- * Only memory held: current batch (≤10 events) + tail (≤5 events).
- */
-function pushEventToBuffers(event, sessionId, requestId, eventBatch, tailBuffer, eventIndexRef) {
-    const entry = { index: eventIndexRef.value, type: event?.type, data: event };
-    eventBatch.push(entry);
-    // Maintain tail buffer (last 5 events for usage extraction)
-    tailBuffer.push(event);
-    if (tailBuffer.length > 5) {
-        tailBuffer.shift();
-    }
-    // Flush batch to DB when threshold reached
-    if (eventBatch.length >= EVENT_BATCH_SIZE) {
-        flushEventBatch(sessionId, requestId, eventBatch, eventIndexRef.value - eventBatch.length + 1);
-        eventBatch.length = 0; // Reset batch (memory freed)
-    }
-    eventIndexRef.value++;
-}
-/**
- * Flush remaining batch + update request completion in DB.
- * Extracts token usage from tail buffer (no need for full events array).
- */
-async function completeStreamSave(sessionId, requestId, requestData, eventBatch, tailBuffer, eventIndexRef, startTime, statusCode, resolved, forwardedModel, originalModel, machineId, request) {
-    // Flush remaining events in batch
-    if (eventBatch.length > 0) {
-        flushEventBatch(sessionId, requestId, eventBatch, eventIndexRef.value - eventBatch.length);
-        eventBatch.length = 0;
-    }
-    requestData.response.completedAt = new Date().toISOString();
-    requestData.response.events = []; // Not needed for streaming, clear to avoid serialization
-    await saveRequestDB(sessionId, requestId, requestData);
-    const usage = extractTokenUsage(tailBuffer);
-    updateRequestTokens(requestId, sessionId, usage);
-    pushNewRequest({
-        requestId,
-        sessionId,
-        compositeId: `${sessionId}:${requestId}`,
-        method: request.method,
-        path: request.url,
-        timestamp: requestData.timestamp,
-        statusCode,
-        responseTime: Date.now() - startTime,
-        model: forwardedModel || originalModel,
-        machineId,
-        userAgent: request.headers['user-agent'],
-        baseUrl: resolved.baseUrl,
-        usage,
-        pendingId: requestId !== request.id ? `${sessionId}:${request.id}` : undefined
-    }, 'completed');
-}
 export function getErrorMessage(error) {
     if (error.code === 'ECONNREFUSED') {
         return 'Target server unreachable';
@@ -227,12 +168,50 @@ export function getErrorMessage(error) {
     }
     return 'Proxy error';
 }
+// SSE 流空闲看门狗：超过该时长无上游数据则销毁流，避免挂死产生僵尸行
+const STREAM_IDLE_TIMEOUT_MS = 120000;
+/**
+ * 给上游 nodeStream 挂空闲看门狗：data 到达重置定时器；
+ * end/close/error 清理定时器；超时且流未销毁时以 idle timeout 销毁。
+ */
+function attachIdleWatchdog(nodeStream) {
+    let timer = null;
+    const clearTimer = () => {
+        if (timer) {
+            clearTimeout(timer);
+            timer = null;
+        }
+    };
+    const resetTimer = () => {
+        clearTimer();
+        timer = setTimeout(() => {
+            if (!nodeStream.destroyed) {
+                nodeStream.destroy(new Error('upstream idle timeout'));
+            }
+        }, STREAM_IDLE_TIMEOUT_MS);
+    };
+    nodeStream.on('data', resetTimer);
+    nodeStream.on('end', clearTimer);
+    nodeStream.on('close', clearTimer);
+    nodeStream.on('error', clearTimer);
+    resetTimer();
+}
 export async function setupProxy(fastify, proxyConfig) {
     // Load full config to get storage.maxDays
     const fullConfig = await loadConfig();
     const maxDays = fullConfig.storage?.maxDays ?? 7;
     // Initialize SQLite database
-    initDatabase();
+    const db = initDatabase();
+    // 回收上次进程遗留的无主 running learn 任务（进程崩溃/被 kill）
+    const recoveredTasks = recoverCrashedTasks(db);
+    if (recoveredTasks > 0) {
+        console.log(`Recovered ${recoveredTasks} crashed learn task(s) from previous run`);
+    }
+    // 回收上次进程遗留的僵尸 SSE 请求行（流中断/挂死未回调）
+    const interruptedRequests = interruptStaleRequestsOnStartup();
+    if (interruptedRequests > 0) {
+        console.log(`Interrupted ${interruptedRequests} stale request(s) from previous run`);
+    }
     // Initialize MCP connections (after DB is ready)
     try {
         await mcpManager.init();
@@ -286,9 +265,12 @@ export async function setupProxy(fastify, proxyConfig) {
     fastify.all('/v1/*', async (request, reply) => {
         const startTime = Date.now();
         const resolved = resolveProxyTarget(proxyConfig);
+        // 下游客户端中止：整个请求作用域建一次，两处上游 fetch 复用
+        const clientSignal = createClientAbortSignal(request.raw);
         const sessionId = request.headers['x-claude-code-session-id'] || 'unknown-session';
         // Replace model based on tier matching (getProxyModel)
         const originalModel = request.body?.model;
+        let rec;
         try {
             const proxyModel = getProxyModel(originalModel, resolved);
             if (proxyModel && request.body && typeof request.body === 'object') {
@@ -312,9 +294,9 @@ export async function setupProxy(fastify, proxyConfig) {
                 (forwardedAuthHeader?.startsWith('Bearer ') ? forwardedAuthHeader.slice(7) : undefined);
             // Log complete request data (Fastify already parsed body)
             const machineId = getMachineIdSync() ?? undefined;
-            const requestData = {
-                requestId: request.id, // Will be updated with cloud x-request-id later
+            const meta = {
                 sessionId,
+                localRequestId: request.id, // Will be replaced with cloud x-request-id row on completion
                 timestamp: new Date().toISOString(),
                 apiKey,
                 proxyApiKey,
@@ -326,34 +308,12 @@ export async function setupProxy(fastify, proxyConfig) {
                     path: request.url,
                     headers: forwardHeaders,
                     body: request.body // Full payload: model, messages, tools, MCP params, etc.
-                },
-                response: {
-                    statusCode: 0, // Will be updated after fetch
-                    events: [] // Only used for JSON responses; streaming uses batch buffer
                 }
             };
-            // SSE event batch buffer: flush to DB every ~10 events to avoid in-memory accumulation
-            // Tail buffer: keep last 5 events for token usage extraction at stream end
-            const eventBatch = [];
-            const tailBuffer = [];
-            const eventIndexRef = { value: 0 };
+            rec = createRequestRecorder(meta);
             request.log.info(`request logged: ${request.id} ${request.method} ${request.url}`);
             // Save initial record + push pending status to frontend
-            await saveRequestDB(sessionId, request.id, requestData);
-            pushNewRequest({
-                requestId: request.id,
-                sessionId,
-                compositeId: `${sessionId}:${request.id}`,
-                method: request.method,
-                path: request.url,
-                timestamp: requestData.timestamp,
-                statusCode: 0,
-                responseTime: 0,
-                model: forwardedModel,
-                machineId,
-                userAgent: request.headers['user-agent'],
-                baseUrl: resolved.baseUrl
-            }, 'pending');
+            await rec.pending();
             // --- OpenAI Compatible Mode ---
             if (resolved.is_openai_compatible) {
                 const openaiRequest = convertClaudeToOpenAIRequest(request.body, forwardedModel);
@@ -366,14 +326,11 @@ export async function setupProxy(fastify, proxyConfig) {
                     method: request.method,
                     headers: openaiHeaders,
                     body: JSON.stringify(openaiRequest),
-                    signal: AbortSignal.timeout(300000)
+                    signal: createTimeoutSignal(300000, clientSignal)
                 });
                 request.log.info(`response status: ${response.status}`);
                 // Extract cloud request ID
                 const cloudRequestId = response.headers.get('x-request-id') || request.id;
-                requestData.requestId = cloudRequestId;
-                requestData.response.statusCode = response.status;
-                requestData.response.headers = Object.fromEntries(response.headers);
                 // Check response type
                 const contentType = response.headers.get('content-type') || '';
                 const isSSE = contentType.includes('text/event-stream');
@@ -386,46 +343,30 @@ export async function setupProxy(fastify, proxyConfig) {
                         const openaiRes = JSON.parse(responseBody);
                         const claudeRes = convertOpenAIToClaudeResponse(openaiRes, forwardedModel || originalModel);
                         claudeResponseBody = JSON.stringify(claudeRes);
-                        requestData.response.events.push(claudeRes);
+                        rec.appendEvent(claudeRes);
                     }
                     catch {
                         claudeResponseBody = responseBody;
                         try {
-                            requestData.response.events.push(JSON.parse(responseBody));
+                            rec.appendEvent(JSON.parse(responseBody));
                         }
                         catch {
-                            requestData.response.events.push({ raw: responseBody });
+                            rec.appendEvent({ raw: responseBody });
                         }
                     }
-                    requestData.response.completedAt = new Date().toISOString();
-                    if (cloudRequestId !== request.id) {
-                        deleteResponseEvents(request.id, sessionId);
-                        deleteRequestRow(request.id, sessionId);
-                    }
-                    await saveRequestDB(sessionId, cloudRequestId, requestData);
-                    const usage = extractTokenUsage(requestData.response.events);
-                    pushNewRequest({
-                        requestId: cloudRequestId,
-                        sessionId,
-                        compositeId: `${sessionId}:${cloudRequestId}`,
-                        method: request.method,
-                        path: request.url,
-                        timestamp: requestData.timestamp,
+                    await rec.complete({
+                        cloudRequestId: cloudRequestId !== request.id ? cloudRequestId : undefined,
                         statusCode: response.status,
-                        responseTime: Date.now() - startTime,
-                        model: forwardedModel,
-                        machineId,
-                        userAgent: request.headers['user-agent'],
-                        baseUrl: resolved.baseUrl,
-                        usage,
-                        pendingId: cloudRequestId !== request.id ? `${sessionId}:${request.id}` : undefined
-                    }, 'completed');
+                        headers: Object.fromEntries(response.headers)
+                    });
                     const elapsed = Date.now() - startTime;
                     request.log.info(`json response (openai): ${cloudRequestId} ${response.status} (${elapsed}ms)`);
                     reply.code(response.status).headers({ 'Content-Type': 'application/json' }).send(claudeResponseBody);
                     return;
                 }
                 // Handle SSE streaming response
+                // 上游响应头已到达：回写真实 status_code（状态仍 pending，保留 spinner）
+                rec.headers({ statusCode: response.status, headers: Object.fromEntries(response.headers) });
                 reply.hijack();
                 const responseHeaders = Object.fromEntries(response.headers);
                 // Replace content-type with Claude SSE format
@@ -433,51 +374,35 @@ export async function setupProxy(fastify, proxyConfig) {
                 reply.raw.writeHead(response.status, responseHeaders);
                 if (response.body) {
                     const nodeStream = Readable.fromWeb(response.body);
+                    // 客户端断开：销毁上游流，finished 回调走中断落库
+                    clientSignal.addEventListener('abort', () => {
+                        if (!nodeStream.destroyed)
+                            nodeStream.destroy(new Error('client disconnected'));
+                    });
+                    attachIdleWatchdog(nodeStream);
                     const transformStream = createStreamTransformer(forwardedModel || originalModel);
-                    // Record SSE events via batch buffer (memory: ≤10 batch + ≤5 tail)
+                    // 旁路解码器：只做事件入库，不介入转发管道；[DONE] 走 onDone 留空，天然不入库
+                    const tapDecoder = createEventDecoder((e) => rec.appendEvent(e));
                     nodeStream.on('data', (chunk) => {
-                        const text = chunk.toString();
-                        const lines = text.split('\n');
-                        for (const line of lines) {
-                            if (line.startsWith('data:')) {
-                                const eventData = line.slice(5).trim();
-                                if (eventData && eventData !== '[DONE]') {
-                                    try {
-                                        const event = JSON.parse(eventData);
-                                        pushEventToBuffers(event, sessionId, cloudRequestId, eventBatch, tailBuffer, eventIndexRef);
-                                    }
-                                    catch {
-                                        // Not valid JSON, skip
-                                    }
-                                }
-                            }
-                        }
+                        tapDecoder.write(chunk);
                     });
                     // Use finished() for reliable stream completion detection
                     finished(nodeStream, async (err) => {
+                        tapDecoder.end();
                         if (err) {
                             request.log.error(`openai stream error: ${err.message}`);
-                        }
-                        // Convert OpenAI stream usage to Claude format for extractTokenUsage compatibility
-                        const lastChunk = tailBuffer.find((e) => e?.usage);
-                        if (lastChunk?.usage) {
-                            const syntheticEvent = {
-                                type: 'message_delta',
-                                usage: {
-                                    output_tokens: lastChunk.usage.completion_tokens || 0
-                                }
-                            };
-                            pushEventToBuffers(syntheticEvent, sessionId, cloudRequestId, eventBatch, tailBuffer, eventIndexRef);
-                        }
-                        if (cloudRequestId !== request.id) {
-                            deleteResponseEvents(request.id, sessionId);
-                            deleteRequestRow(request.id, sessionId);
+                            rec.interrupt({ statusCode: response.status, error: `stream interrupted: ${err.message}` });
+                            return;
                         }
                         // Flush remaining batch + complete request
                         try {
-                            await completeStreamSave(sessionId, cloudRequestId, requestData, eventBatch, tailBuffer, eventIndexRef, startTime, response.status, resolved, forwardedModel, originalModel, machineId, request);
+                            await rec.complete({
+                                cloudRequestId: cloudRequestId !== request.id ? cloudRequestId : undefined,
+                                statusCode: response.status,
+                                headers: Object.fromEntries(response.headers)
+                            });
                             const elapsed = Date.now() - startTime;
-                            request.log.info(`stream completed (openai): ${cloudRequestId} ${response.status} (${elapsed}ms, ${eventIndexRef.value} events)`);
+                            request.log.info(`stream completed (openai): ${cloudRequestId} ${response.status} (${elapsed}ms)`);
                         }
                         catch (dbErr) {
                             request.log.error(`openai stream save failed: ${dbErr.message}`);
@@ -498,15 +423,11 @@ export async function setupProxy(fastify, proxyConfig) {
                 method: request.method,
                 headers: forwardHeaders,
                 body: request.method !== 'GET' ? JSON.stringify(request.body) : undefined,
-                signal: AbortSignal.timeout(300000)
+                signal: createTimeoutSignal(300000, clientSignal)
             });
             request.log.info(`response status: ${response.status}`);
             // Extract cloud request ID from response header
             const cloudRequestId = response.headers.get('x-request-id') || request.id;
-            requestData.requestId = cloudRequestId;
-            // Update response status in requestData
-            requestData.response.statusCode = response.status;
-            requestData.response.headers = Object.fromEntries(response.headers);
             // Check response type: SSE stream or JSON
             const contentType = response.headers.get('content-type') || '';
             const isSSE = contentType.includes('text/event-stream');
@@ -517,35 +438,16 @@ export async function setupProxy(fastify, proxyConfig) {
                 // Record JSON body as a single event
                 try {
                     const jsonBody = JSON.parse(responseBody);
-                    requestData.response.events.push(jsonBody);
+                    rec.appendEvent(jsonBody);
                 }
                 catch {
-                    requestData.response.events.push({ raw: responseBody });
-                }
-                requestData.response.completedAt = new Date().toISOString();
-                // Remove pending row if cloudRequestId differs from request.id
-                if (cloudRequestId !== request.id) {
-                    deleteResponseEvents(request.id, sessionId);
-                    deleteRequestRow(request.id, sessionId);
+                    rec.appendEvent({ raw: responseBody });
                 }
-                await saveRequestDB(sessionId, cloudRequestId, requestData);
-                const usage = extractTokenUsage(requestData.response.events);
-                pushNewRequest({
-                    requestId: cloudRequestId,
-                    sessionId,
-                    compositeId: `${sessionId}:${cloudRequestId}`,
-                    method: request.method,
-                    path: request.url,
-                    timestamp: requestData.timestamp,
+                await rec.complete({
+                    cloudRequestId: cloudRequestId !== request.id ? cloudRequestId : undefined,
                     statusCode: response.status,
-                    responseTime: Date.now() - startTime,
-                    model: forwardedModel,
-                    machineId,
-                    userAgent: request.headers['user-agent'],
-                    baseUrl: resolved.baseUrl,
-                    usage,
-                    pendingId: cloudRequestId !== request.id ? `${sessionId}:${request.id}` : undefined
-                }, 'completed');
+                    headers: Object.fromEntries(response.headers)
+                });
                 const elapsed = Date.now() - startTime;
                 request.log.info(`json response: ${cloudRequestId} ${response.status} (${elapsed}ms)`);
                 reply.code(response.status);
@@ -554,6 +456,8 @@ export async function setupProxy(fastify, proxyConfig) {
                 return;
             }
             // Hijack reply to take full control of SSE stream
+            // 上游响应头已到达：回写真实 status_code（状态仍 pending，保留 spinner）
+            rec.headers({ statusCode: response.status, headers: Object.fromEntries(response.headers) });
             reply.hijack();
             // Write response headers (preserve original from Aliyun)
             const responseHeaders = Object.fromEntries(response.headers);
@@ -562,38 +466,33 @@ export async function setupProxy(fastify, proxyConfig) {
             if (response.body) {
                 // Convert Web ReadableStream to Node Readable stream
                 const nodeStream = Readable.fromWeb(response.body);
-                // Record SSE events via batch buffer (memory: ≤10 batch + ≤5 tail)
+                // 客户端断开：销毁上游流，finished 回调走中断落库
+                clientSignal.addEventListener('abort', () => {
+                    if (!nodeStream.destroyed)
+                        nodeStream.destroy(new Error('client disconnected'));
+                });
+                attachIdleWatchdog(nodeStream);
+                // 旁路解码器：只做事件入库，不介入转发管道；[DONE] 走 onDone 留空，天然不入库
+                const tapDecoder = createEventDecoder((e) => rec.appendEvent(e));
                 nodeStream.on('data', (chunk) => {
-                    const text = chunk.toString();
-                    const lines = text.split('\n');
-                    for (const line of lines) {
-                        if (line.startsWith('data:')) {
-                            const eventData = line.slice(5).trim();
-                            if (eventData && eventData !== '[DONE]') {
-                                try {
-                                    const event = JSON.parse(eventData);
-                                    pushEventToBuffers(event, sessionId, cloudRequestId, eventBatch, tailBuffer, eventIndexRef);
-                                }
-                                catch {
-                                    // Not valid JSON, skip
-                                }
-                            }
-                        }
-                    }
+                    tapDecoder.write(chunk);
                 });
                 // Save request data when stream completes
                 finished(nodeStream, async (err) => {
+                    tapDecoder.end();
                     if (err) {
                         request.log.error(`claude stream error: ${err.message}`);
-                    }
-                    if (cloudRequestId !== request.id) {
-                        deleteResponseEvents(request.id, sessionId);
-                        deleteRequestRow(request.id, sessionId);
+                        rec.interrupt({ statusCode: response.status, error: `stream interrupted: ${err.message}` });
+                        return;
                     }
                     try {
-                        await completeStreamSave(sessionId, cloudRequestId, requestData, eventBatch, tailBuffer, eventIndexRef, startTime, response.status, resolved, forwardedModel, originalModel, machineId, request);
+                        await rec.complete({
+                            cloudRequestId: cloudRequestId !== request.id ? cloudRequestId : undefined,
+                            statusCode: response.status,
+                            headers: Object.fromEntries(response.headers)
+                        });
                         const elapsed = Date.now() - startTime;
-                        request.log.info(`stream completed: ${cloudRequestId} ${response.status} (${elapsed}ms, ${eventIndexRef.value} events)`);
+                        request.log.info(`stream completed: ${cloudRequestId} ${response.status} (${elapsed}ms)`);
                     }
                     catch (dbErr) {
                         request.log.error(`stream save failed: ${dbErr.message}`);
@@ -610,22 +509,27 @@ export async function setupProxy(fastify, proxyConfig) {
                 errorMessage: error.message,
                 errorCode: error.code || 'unknown'
             }, 'proxy error');
-            // Fetch failed — no cloudRequestId available, use Fastify request.id
-            pushNewRequest({
-                requestId: request.id,
-                sessionId,
-                compositeId: `${sessionId}:${request.id}`,
-                method: request.method,
-                path: request.url,
-                timestamp: new Date().toISOString(),
-                statusCode: getErrorStatusCode(error),
-                responseTime: Date.now() - startTime,
-                model: originalModel,
-                machineId: getMachineIdSync() ?? undefined,
-                userAgent: request.headers['user-agent'],
-                baseUrl: resolved.baseUrl
-            }, 'completed');
             const statusCode = getErrorStatusCode(error);
+            if (rec) {
+                rec.fail({ statusCode, error: getErrorMessage(error) });
+            }
+            else {
+                // pending() 之前就抛错（header/meta 准备阶段），无 DB 行可更新，仅推送
+                pushNewRequest({
+                    requestId: request.id,
+                    sessionId,
+                    compositeId: `${sessionId}:${request.id}`,
+                    method: request.method,
+                    path: request.url,
+                    timestamp: new Date().toISOString(),
+                    statusCode,
+                    responseTime: Date.now() - startTime,
+                    model: originalModel,
+                    machineId: getMachineIdSync() ?? undefined,
+                    userAgent: request.headers['user-agent'],
+                    baseUrl: resolved.baseUrl
+                }, 'completed');
+            }
             reply.code(statusCode).send({
                 error: getErrorMessage(error),
                 requestId: request.id