@cc-monitoring/proxy-core 0.4.0 → 0.4.2

package/dist/utils/request-writer.js

@@ -20,7 +20,7 @@ function flushEventBatch(sessionId, requestId, batch, startIndex) {
     const db = getDatabase();
     ensureStmtCacheValid(db);
     if (!insertEventStmt) {
-        insertEventStmt = db.prepare('INSERT INTO response_events (request_id, session_id, event_index, event_type, event_data) VALUES (?, ?, ?, ?, ?)');
+        insertEventStmt = db.prepare('INSERT OR IGNORE INTO response_events (request_id, session_id, event_index, event_type, event_data) VALUES (?, ?, ?, ?, ?)');
     }
     for (let i = 0; i < batch.length; i++) {
         const event = batch[i];
@@ -33,6 +33,118 @@ function extractUsage(events) {
     const messageDelta = events.find((e) => e?.type === 'message_delta');
     return messageDelta?.usage;
 }
+// --- 流完整性判定与 usage 重建（纯函数，事件数组入参）---
+/**
+ * 判断事件集合是否构成一条完整返回的流。
+ * Claude：出现 type === 'message_stop'。
+ * OpenAI 兼容：chunk 内 choices[0].finish_reason 非空
+ * （[DONE] 不入库，不能作为 DB 判定依据）。
+ */
+export function isStreamCompleteEvents(events) {
+    if (!Array.isArray(events))
+        return false;
+    return events.some(e => {
+        if (!e || typeof e !== 'object')
+            return false;
+        if (e.type === 'message_stop')
+            return true;
+        const finishReason = e.choices?.[0]?.finish_reason;
+        return finishReason !== undefined && finishReason !== null && finishReason !== '';
+    });
+}
+/**
+ * 从事件集合按字段合并重建 usage，不做整体覆盖。
+ * Claude：message_start.usage（input_tokens / cache_*）为底；
+ * message_delta.usage 的字段优先（output_tokens 为主，cache 字段也合入）。
+ * OpenAI：取带 .usage 的 chunk（prompt_tokens/completion_tokens）。
+ */
+export function reconstructUsageFromEvents(events) {
+    if (!Array.isArray(events))
+        return undefined;
+    const usage = {};
+    let found = false;
+    const merge = (field, value, precedence) => {
+        if (typeof value !== 'number')
+            return;
+        if (precedence || usage[field] === undefined) {
+            usage[field] = value;
+            found = true;
+        }
+    };
+    for (const e of events) {
+        if (!e || typeof e !== 'object')
+            continue;
+        if (e.type === 'message_start' || e.type === 'message_delta') {
+            const u = e.usage;
+            if (u && typeof u === 'object') {
+                const isDelta = e.type === 'message_delta';
+                merge('input_tokens', u.input_tokens, isDelta);
+                merge('output_tokens', u.output_tokens, isDelta);
+                merge('cache_read_input_tokens', u.cache_read_input_tokens, isDelta);
+                merge('cache_creation_input_tokens', u.cache_creation_input_tokens, isDelta);
+            }
+        }
+        // OpenAI 兼容：带 usage 的 chunk（event_type 通常为 NULL）
+        if (e.usage && typeof e.usage === 'object' && e.type !== 'message_start' && e.type !== 'message_delta') {
+            const u = e.usage;
+            merge('input_tokens', u.input_tokens ?? u.prompt_tokens, false);
+            merge('output_tokens', u.output_tokens ?? u.completion_tokens, false);
+        }
+    }
+    return found ? usage : undefined;
+}
+// 定向加载重建所需事件，禁止全量加载事件流（实测有 8000+ 行）：
+// 1) Claude 锚点事件（message_start/message_delta/message_stop，通常仅数行）
+// 2) 尾部 10 条事件（覆盖 OpenAI 的 finish_reason / usage，其 event_type 可能为 NULL）
+function loadTargetedEvents(db, requestId, sessionId) {
+    const anchorRows = db.prepare(`
+    SELECT event_data FROM response_events
+    WHERE request_id = ? AND session_id = ?
+      AND event_type IN ('message_start', 'message_delta', 'message_stop')
+    ORDER BY event_index ASC
+  `).all(requestId, sessionId);
+    const tailRows = db.prepare(`
+    SELECT event_data FROM response_events
+    WHERE request_id = ? AND session_id = ?
+    ORDER BY event_index DESC
+    LIMIT 10
+  `).all(requestId, sessionId);
+    const seen = new Set();
+    const events = [];
+    for (const row of [...anchorRows, ...tailRows]) {
+        if (seen.has(row.event_data))
+            continue;
+        seen.add(row.event_data);
+        try {
+            events.push(JSON.parse(row.event_data));
+        }
+        catch {
+            // 忽略损坏行
+        }
+    }
+    return events;
+}
+// 按字段回填 token 列（只补 NULL 列），返回是否有列被更新
+function fillTokenColumns(db, requestId, sessionId, usage) {
+    const current = db.prepare(`
+    SELECT input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens
+    FROM requests WHERE request_id = ? AND session_id = ?
+  `).get(requestId, sessionId);
+    if (!current)
+        return false;
+    const pairs = [
+        ['input_tokens', usage.input_tokens],
+        ['output_tokens', usage.output_tokens],
+        ['cache_read_tokens', usage.cache_read_input_tokens],
+        ['cache_creation_tokens', usage.cache_creation_input_tokens]
+    ];
+    const updatable = pairs.filter(([col, v]) => current[col] === null && typeof v === 'number');
+    if (updatable.length === 0)
+        return false;
+    const setSql = updatable.map(([col]) => `${col} = ?`).join(', ');
+    db.prepare(`UPDATE requests SET ${setSql} WHERE request_id = ? AND session_id = ?`).run(...updatable.map(([, v]) => v), requestId, sessionId);
+    return true;
+}
 function writeCompletedRow(args) {
     const db = getDatabase();
     const { meta } = args;
@@ -76,6 +188,13 @@ export function createRequestRecorder(meta) {
     let eventIndex = 0;
     const startTime = Date.now();
     let pendingCalled = false;
+    // recorder 生命周期开始前的事件行下界。complete() 把事件从 localRequestId 迁到
+    // cloudRequestId 时只允许迁移 id 大于它的行：request_id（req-N 计数器）在进程
+    // 重启后可能复用，同 localRequestId 的历史残留事件属于别的请求，不能被一起迁走。
+    const startRow = getDatabase()
+        .prepare('SELECT MAX(id) AS m FROM response_events')
+        .get();
+    const startEventRowId = startRow.m ?? 0;
     function flushRemaining() {
         if (eventBatch.length > 0) {
             flushEventBatch(meta.sessionId, meta.localRequestId, eventBatch, eventIndex - eventBatch.length);
@@ -141,7 +260,9 @@ export function createRequestRecorder(meta) {
             const db = getDatabase();
             if (finalId !== meta.localRequestId) {
                 const replace = db.transaction(() => {
-                    db.prepare('UPDATE response_events SET request_id = ? WHERE request_id = ? AND session_id = ?').run(finalId, meta.localRequestId, meta.sessionId);
+                    // id > startEventRowId：只迁本 recorder 创建后写入的事件，
+                    // 不迁同 localRequestId 的历史残留事件
+                    db.prepare('UPDATE response_events SET request_id = ? WHERE request_id = ? AND session_id = ? AND id > ?').run(finalId, meta.localRequestId, meta.sessionId, startEventRowId);
                     writeCompletedRow({
                         meta, requestId: finalId, statusCode, responseTime, headers, usage, completedAt
                     });
@@ -156,13 +277,20 @@ export function createRequestRecorder(meta) {
             }
             pushNewRequest(buildSummary(meta, finalId, statusCode, responseTime, usage, finalId !== meta.localRequestId), 'completed');
         },
-        // 流被截断/挂死：保留真实 status_code，通过 closed_at + error 标记中断
+        // 流被截断/挂死：保留真实 status_code，通过 closed_at + error 标记中断；
+        // 同时通过定向 SQL（锚点事件 + 尾部事件）重建 usage 并按字段回填 token
         interrupt({ statusCode, error }) {
             flushRemaining();
+            const db = getDatabase();
             const responseTime = Date.now() - startTime;
             const closedAt = new Date().toISOString();
-            getDatabase().prepare('UPDATE requests SET status_code = ?, response_time = ?, closed_at = ?, error = ? WHERE request_id = ? AND session_id = ?').run(statusCode, responseTime, closedAt, error, meta.localRequestId, meta.sessionId);
-            pushNewRequest(buildSummary(meta, meta.localRequestId, statusCode, responseTime, undefined, false, closedAt), 'completed');
+            db.prepare('UPDATE requests SET status_code = ?, response_time = ?, closed_at = ?, error = ? WHERE request_id = ? AND session_id = ?').run(statusCode, responseTime, closedAt, error, meta.localRequestId, meta.sessionId);
+            const events = loadTargetedEvents(db, meta.localRequestId, meta.sessionId);
+            const usage = reconstructUsageFromEvents(events);
+            if (usage) {
+                fillTokenColumns(db, meta.localRequestId, meta.sessionId, usage);
+            }
+            pushNewRequest(buildSummary(meta, meta.localRequestId, statusCode, responseTime, usage, false, closedAt), 'completed');
         },
         fail({ statusCode, error }) {
             const responseTime = Date.now() - startTime;
@@ -275,7 +403,7 @@ export async function listRequestDB(limit = 1000, before) {
     SELECT
       request_id, session_id, timestamp, method, path, status_code, response_time,
       api_key, proxy_api_key, base_url, model, machine_id, user_agent,
-      input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, closed_at
+      input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, closed_at, error
     FROM requests
     ${whereClause}
     ORDER BY timestamp DESC
@@ -305,7 +433,8 @@ export async function listRequestDB(limit = 1000, before) {
                 cache_read_input_tokens: row.cache_read_tokens,
                 cache_creation_input_tokens: row.cache_creation_tokens
             } : undefined,
-            closedAt: row.closed_at || undefined
+            closedAt: row.closed_at || undefined,
+            error: row.error || undefined
         })),
         hasMore
     };
@@ -322,23 +451,80 @@ export async function cleanOldRequestsDB(daysToKeep = 7) {
     // Delete response_events first (foreign data), then requests
     db.prepare(`DELETE FROM response_events WHERE request_id IN (SELECT request_id FROM requests WHERE timestamp < ?)`).run(cutoff);
     db.prepare(`DELETE FROM requests WHERE timestamp < ?`).run(cutoff);
+    // 例行还页：auto_vacuum 未开启时为 no-op；经 ccm vacuum 开启
+    // INCREMENTAL 后，删除产生的空闲页在此廉价归还文件系统。
+    db.pragma('incremental_vacuum');
 }
-// 启动时回收上次进程遗留的僵尸 SSE 行：未完成、未关闭且超龄，逐行标记为中断
-// （status_code 保持原值不动），返回处理行数
+// 启动时回收上次进程遗留的僵尸 SSE 行：未完成、未关闭且超龄，逐行分流
+// （status_code 保持原值不动）——事件流完整（message_stop / OpenAI finish_reason）
+// 的走完成语义：写 completed_at，不写 closed_at/error；不完整的维持
+// closed_at + 'interrupted: proxy restarted'。两条路径都按字段回填 token 列。
 export function interruptStaleRequestsOnStartup(maxAgeMs = 5 * 60000) {
     const db = initDatabase();
     const cutoff = new Date(Date.now() - maxAgeMs).toISOString();
     const rows = db.prepare(`
-    SELECT request_id, session_id
+    SELECT request_id, session_id, timestamp, response_time
     FROM requests
     WHERE completed_at IS NULL AND closed_at IS NULL AND timestamp < ?
   `).all(cutoff);
-    const closedAt = new Date().toISOString();
-    const stmt = db.prepare('UPDATE requests SET closed_at = ?, error = ? WHERE request_id = ? AND session_id = ?');
+    let completed = 0;
+    let interrupted = 0;
+    const nowIso = new Date().toISOString();
     for (const row of rows) {
-        stmt.run(closedAt, 'interrupted: proxy restarted', row.request_id, row.session_id);
+        const events = loadTargetedEvents(db, row.request_id, row.session_id);
+        const usage = reconstructUsageFromEvents(events);
+        if (isStreamCompleteEvents(events)) {
+            // response_time 原值为 0（pending 占位）时，用 timestamp→现在给合理值
+            const responseTime = row.response_time > 0
+                ? row.response_time
+                : Math.max(0, Date.now() - Date.parse(row.timestamp));
+            db.prepare('UPDATE requests SET completed_at = ?, response_time = ? WHERE request_id = ? AND session_id = ?').run(nowIso, responseTime, row.request_id, row.session_id);
+            completed += 1;
+        }
+        else {
+            db.prepare('UPDATE requests SET closed_at = ?, error = ? WHERE request_id = ? AND session_id = ?').run(nowIso, 'interrupted: proxy restarted', row.request_id, row.session_id);
+            interrupted += 1;
+        }
+        if (usage) {
+            fillTokenColumns(db, row.request_id, row.session_id, usage);
+        }
+    }
+    return { completed, interrupted };
+}
+// 存量回填（供 CLI repair-requests 使用），幂等可重跑：
+// 扫描 closed_at 非空的行——事件流完整且 completed_at 为空的：清 closed_at/error、
+// 写 completed_at（completed_at 已非空的跳过）；对 token 全 NULL 但事件可恢复
+// usage 的行按字段补 token。同样只走定向 SQL。
+export function repairClosedRequests() {
+    const db = initDatabase();
+    const rows = db.prepare(`
+    SELECT
+      request_id, session_id, completed_at,
+      input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens
+    FROM requests
+    WHERE closed_at IS NOT NULL
+  `).all();
+    let recovered = 0;
+    let tokensFilled = 0;
+    const nowIso = new Date().toISOString();
+    for (const row of rows) {
+        const events = loadTargetedEvents(db, row.request_id, row.session_id);
+        if (row.completed_at === null && isStreamCompleteEvents(events)) {
+            db.prepare('UPDATE requests SET closed_at = NULL, error = NULL, completed_at = ? WHERE request_id = ? AND session_id = ?').run(nowIso, row.request_id, row.session_id);
+            recovered += 1;
+        }
+        const allTokensNull = row.input_tokens === null &&
+            row.output_tokens === null &&
+            row.cache_read_tokens === null &&
+            row.cache_creation_tokens === null;
+        if (allTokensNull) {
+            const usage = reconstructUsageFromEvents(events);
+            if (usage && fillTokenColumns(db, row.request_id, row.session_id, usage)) {
+                tokensFilled += 1;
+            }
+        }
     }
-    return rows.length;
+    return { recovered, tokensFilled };
 }
 // Load session history for multi-turn chat resume
 // Returns the last request's messages + assistant response, to prepend to new messages

package/dist/utils/stats-aggregator.d.ts

@@ -16,6 +16,6 @@ export interface TodayStats {
     request_count: number;
     models: Record<string, number>;
 }
-export declare function aggregateStats(): void;
+export declare function aggregateStats(maxDays?: number): void;
 export declare function getStatsData(since?: string): StatsData;
 export declare function getTodayStats(since: string): TodayStats;

package/dist/utils/stats-aggregator.js

@@ -1,27 +1,47 @@
 import { initDatabase } from './db.js';
+// 共用 token 聚合列：小时聚合与今日 totals 同口径，避免列漂移
+const TOKEN_AGG_COLUMNS = `
+  SUM(input_tokens) as input_tokens,
+  SUM(output_tokens) as output_tokens,
+  SUM(cache_read_tokens) as cache_read_tokens,
+  SUM(cache_creation_tokens) as cache_creation_tokens,
+  COUNT(*) as request_count
+`;
+// Build the UTC hourly key ('YYYY-MM-DD-HH') used as stats.period_key
+function toHourKey(date) {
+    return `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, '0')}-${String(date.getUTCDate()).padStart(2, '0')}-${String(date.getUTCHours()).padStart(2, '0')}`;
+}
 // Aggregate stats from requests table (called periodically)
 // Only stores hourly (UTC) entries — daily aggregation is done on the frontend
 // using the user's local timezone.
-export function aggregateStats() {
+// maxDays: hourly retention window in days. A positive integer shrinks the
+// window (capped at 30); undefined / <=0 falls back to the default 30 days.
+export function aggregateStats(maxDays) {
     const db = initDatabase();
     const now = new Date();
     const nowIso = now.toISOString();
-    // 30 days of hourly data = 720 entries
-    const hourlyCutoff = new Date(now.getTime() - 30 * 86400000).toISOString();
+    // windowDays days of hourly data (30 days = 720 entries)
+    const windowDays = Number.isInteger(maxDays) && maxDays > 0
+        ? Math.min(30, maxDays)
+        : 30;
+    const hourlyCutoff = new Date(now.getTime() - windowDays * 86400000).toISOString();
     // Aggregate hourly stats using SQLite's built-in strftime (UTC)
     const hourlyRows = db.prepare(`
     SELECT
       strftime('%Y-%m-%d-%H', timestamp) as hour,
-      SUM(input_tokens) as input_tokens,
-      SUM(output_tokens) as output_tokens,
-      SUM(cache_read_tokens) as cache_read_tokens,
-      SUM(cache_creation_tokens) as cache_creation_tokens,
-      COUNT(*) as request_count
+      ${TOKEN_AGG_COLUMNS}
     FROM requests
     WHERE timestamp >= ?
-      AND closed_at IS NULL
     GROUP BY hour
   `).all(hourlyCutoff);
+    // Recompute the whole window atomically: drop every hourly snapshot inside
+    // the window first (source rows may already have been deleted by cleanup, so
+    // an upsert-only pass would leave stale snapshots frozen forever), then
+    // re-insert from the requests table.
+    // Cutoff key uses UTC methods to match the UTC keys stored in the DB.
+    // The two deletes (< key / >= key) together clear every hourly row.
+    const cutoffKey = toHourKey(new Date(hourlyCutoff));
+    const deleteWindow = db.prepare(`DELETE FROM stats WHERE period = 'hourly' AND period_key >= ?`);
     // Upsert hourly stats
     const upsertHourly = db.prepare(`
     INSERT OR REPLACE INTO stats (
@@ -29,16 +49,19 @@ export function aggregateStats() {
       cache_read_tokens, cache_creation_tokens, request_count, updated_at
     ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
   `);
-    for (const row of hourlyRows) {
-        upsertHourly.run('hourly', row.hour, row.input_tokens || 0, row.output_tokens || 0, row.cache_read_tokens || 0, row.cache_creation_tokens || 0, row.request_count, nowIso);
-    }
-    // Clean old hourly entries: compute cutoff key using UTC methods
-    // to match the UTC keys stored in the DB
-    const cutoffDate = new Date(hourlyCutoff);
-    const cutoffKey = `${cutoffDate.getUTCFullYear()}-${String(cutoffDate.getUTCMonth() + 1).padStart(2, '0')}-${String(cutoffDate.getUTCDate()).padStart(2, '0')}-${String(cutoffDate.getUTCHours()).padStart(2, '0')}`;
-    db.prepare(`DELETE FROM stats WHERE period = 'hourly' AND period_key < ?`).run(cutoffKey);
+    // Clean hourly entries outside the recomputation window
+    const deleteOldKeys = db.prepare(`DELETE FROM stats WHERE period = 'hourly' AND period_key < ?`);
     // Clean ALL daily entries (legacy data from old +8h approach)
-    db.prepare(`DELETE FROM stats WHERE period = 'daily'`).run();
+    const deleteDaily = db.prepare(`DELETE FROM stats WHERE period = 'daily'`);
+    const recompute = db.transaction(() => {
+        deleteWindow.run(cutoffKey);
+        for (const row of hourlyRows) {
+            upsertHourly.run('hourly', row.hour, row.input_tokens || 0, row.output_tokens || 0, row.cache_read_tokens || 0, row.cache_creation_tokens || 0, row.request_count, nowIso);
+        }
+        deleteOldKeys.run(cutoffKey);
+        deleteDaily.run();
+    });
+    recompute();
     // WAL checkpoint to keep WAL file size bounded
     db.pragma('wal_checkpoint(TRUNCATE)');
 }
@@ -48,8 +71,7 @@ export function getStatsData(since) {
     let hourlyRows;
     if (since) {
         // Convert since (ISO UTC) to UTC hourly key format for filtering
-        const sinceDate = new Date(since);
-        const sinceKey = `${sinceDate.getUTCFullYear()}-${String(sinceDate.getUTCMonth() + 1).padStart(2, '0')}-${String(sinceDate.getUTCDate()).padStart(2, '0')}-${String(sinceDate.getUTCHours()).padStart(2, '0')}`;
+        const sinceKey = toHourKey(new Date(since));
         hourlyRows = db.prepare(`
       SELECT * FROM stats WHERE period = 'hourly' AND period_key >= ? ORDER BY period_key ASC
     `).all(sinceKey);
@@ -83,22 +105,17 @@ export function getStatsData(since) {
 export function getTodayStats(since) {
     const db = initDatabase();
     // Totals: SUM tokens + COUNT requests since local midnight
+    // 多选的 cache_creation_tokens 在返回映射时忽略
     const totalRow = db.prepare(`
-    SELECT
-      SUM(input_tokens) as input_tokens,
-      SUM(output_tokens) as output_tokens,
-      SUM(cache_read_tokens) as cache_read_tokens,
-      COUNT(*) as request_count
+    SELECT ${TOKEN_AGG_COLUMNS}
     FROM requests
     WHERE timestamp >= ?
-      AND closed_at IS NULL
   `).get(since);
     // Per-model usage count since local midnight
     const modelRows = db.prepare(`
     SELECT model, COUNT(*) as count
     FROM requests
     WHERE timestamp >= ?
-      AND closed_at IS NULL
       AND model IS NOT NULL AND model != ''
     GROUP BY model
     ORDER BY count DESC

package/ecosystem.config.cjs

@@ -15,7 +15,8 @@ module.exports = {
     exec_mode: 'fork',  // Use fork mode for ESM compatibility
     autorestart: true,
     watch: false,
-    max_memory_restart: '200M',
+    // 进程 baseline ~184MB 贴近 200M 线导致频繁 OOM kill，内存增长另排查
+    max_memory_restart: '600M',
     env: {
       NODE_ENV: 'production',
       CONFIG_PATH: configPath,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cc-monitoring/proxy-core",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Claude Code 代理工具 + Token 消耗监控面板",
   "keywords": [
     "claude-code-proxy",