cx-viewer 1.0.0

package/lib/log-watcher.js

@@ -0,0 +1,245 @@
+import { readFileSync, existsSync, watchFile, unwatchFile, openSync, readSync, closeSync, statSync } from 'node:fs';
+import { isMainAgentEntry, extractCachedContent } from './kv-cache-analyzer.js';
+import { buildContextWindowEvent, getContextSizeForModel } from './context-watcher.js';
+import { reconstructEntries, createIncrementalReconstructor } from './delta-reconstructor.js';
+import { countLogEntries, streamReconstructedEntries } from './log-stream.js';
+
+// 跟踪所有被 watch 的日志文件
+const watchedFiles = new Map();
+
+/**
+ * Read and parse a JSONL log file.
+ * @param {string} logFile - absolute path to the log file
+ * @returns {Array} parsed and deduplicated entries
+ */
+export function readLogFile(logFile) {
+  if (!existsSync(logFile)) {
+    return [];
+  }
+
+  try {
+    const content = readFileSync(logFile, 'utf-8');
+    const entries = content.split('\n---\n').filter(line => line.trim());
+    const parsed = entries.map(entry => {
+      try {
+        return JSON.parse(entry);
+      } catch {
+        return null;
+      }
+    }).filter(Boolean);
+    // 去重：同一 timestamp+url 的条目，后出现的（带 response）覆盖先出现的（在途）
+    const map = new Map();
+    for (const entry of parsed) {
+      const key = `${entry.timestamp}|${entry.url}`;
+      map.set(key, entry);
+    }
+    return reconstructEntries(Array.from(map.values()));
+  } catch (err) {
+    console.error('Error reading log file:', err);
+    return [];
+  }
+}
+
+/**
+ * Send an SSE entry to all connected clients.
+ * @param {Array} clients - SSE client array
+ * @param {object} entry - parsed log entry
+ */
+export function sendToClients(clients, entry) {
+  clients.forEach(client => {
+    try {
+      client.write(`data: ${JSON.stringify(entry)}\n\n`);
+    } catch (err) {
+      // Client disconnected
+    }
+  });
+}
+
+/**
+ * Send a named SSE event to all connected clients.
+ * @param {Array} clients - SSE client array
+ * @param {string} eventName - SSE event name
+ * @param {object} data - event payload
+ */
+export function sendEventToClients(clients, eventName, data) {
+  clients.forEach(client => {
+    try {
+      client.write(`event: ${eventName}\ndata: ${JSON.stringify(data)}\n\n`);
+    } catch (err) {}
+  });
+}
+
+/**
+ * Watch a log file for changes and broadcast new entries.
+ * @param {object} opts
+ * @param {string} opts.logFile - log file to watch
+ * @param {Array} opts.clients - SSE clients array
+ * @param {Function} opts.getCodexPid - returns Codex process PID
+ * @param {Function} opts.runParallelHook - plugin hook runner
+ * @param {Function} opts.notifyStatsWorker - stats worker notifier
+ * @param {Function} opts.getLogFile - returns current LOG_FILE value
+ */
+export function watchLogFile(opts) {
+  const { logFile, clients, getCodexPid, runParallelHook, notifyStatsWorker, getLogFile } = opts;
+  if (watchedFiles.has(logFile)) return;
+
+  // Track byte offset instead of string length — avoids full-file re-read on every poll
+  let lastByteOffset = 0;
+  let pendingTail = ''; // incomplete entry carried across polls
+  // Delta storage: 增量重建器，用于逐条重建 mainAgent delta 条目
+  const _reconstructor = createIncrementalReconstructor();
+  try {
+    if (existsSync(logFile)) {
+      lastByteOffset = statSync(logFile).size;
+    }
+  } catch {}
+
+  watchedFiles.set(logFile, true);
+  watchFile(logFile, { interval: 500 }, () => {
+    try {
+      const currentSize = statSync(logFile).size;
+
+      // File truncated (rotation or clear) — reset offset and check rotation immediately
+      if (currentSize < lastByteOffset) {
+        lastByteOffset = 0;
+        pendingTail = '';
+        _reconstructor.reset();
+
+        // 文件被清空可能是轮转信号，立即检查是否已切换到新文件
+        const currentLogFile = getLogFile();
+        if (currentLogFile !== logFile && !watchedFiles.has(currentLogFile)) {
+          unwatchFile(logFile);
+          watchedFiles.delete(logFile);
+
+          // 流式分段广播，避免全量加载 OOM
+          const rotTotal = countLogEntries(currentLogFile);
+          clients.forEach(client => {
+            try { client.write(`event: load_start\ndata: ${JSON.stringify({ total: rotTotal, incremental: false })}\n\n`); } catch { }
+          });
+          streamReconstructedEntries(currentLogFile, (segment) => {
+            const data = JSON.stringify(segment);
+            clients.forEach(client => {
+              try { client.write(`event: load_chunk\ndata: ${data}\n\n`); } catch { }
+            });
+          });
+          clients.forEach(client => {
+            try { client.write(`event: load_end\ndata: {}\n\n`); } catch { }
+          });
+          watchLogFile({ ...opts, logFile: currentLogFile });
+          return;
+        }
+      }
+
+      if (currentSize <= lastByteOffset) return;
+
+      // Read only the new bytes
+      const bytesToRead = currentSize - lastByteOffset;
+      const buf = Buffer.alloc(bytesToRead);
+      const fd = openSync(logFile, 'r');
+      try {
+        readSync(fd, buf, 0, bytesToRead, lastByteOffset);
+      } finally {
+        closeSync(fd);
+      }
+      lastByteOffset = currentSize;
+
+      const raw = pendingTail + buf.toString('utf-8');
+      const parts = raw.split('\n---\n');
+
+      // Last part may be incomplete — keep it for next poll
+      pendingTail = parts.pop() || '';
+
+      // If there's only the tail and no complete entries, check if tail is a complete entry
+      // (happens when the file ends without a trailing \n---\n)
+      if (parts.length === 0 && pendingTail.trim()) {
+        try {
+          JSON.parse(pendingTail);
+          // Valid JSON — treat as complete entry
+          parts.push(pendingTail);
+          pendingTail = '';
+        } catch {
+          // Incomplete — keep in pendingTail for next poll
+        }
+      }
+
+      const validParts = parts.filter(p => p.trim());
+      if (validParts.length > 0) {
+        validParts.forEach(entry => {
+          try {
+            const parsed = JSON.parse(entry);
+            if (!parsed.pid) {
+              parsed.pid = getCodexPid();
+            }
+            // Delta storage: reconstruct before push — 确保前端收到完整 messages
+            _reconstructor.reconstruct(parsed);
+            sendToClients(clients, parsed);
+            runParallelHook('onNewEntry', parsed).catch(() => {});
+            if (isMainAgentEntry(parsed) && !parsed.inProgress) {
+              const cached = extractCachedContent(parsed);
+              if (cached) {
+                sendEventToClients(clients, 'kv_cache_content', cached);
+              }
+              const usage = parsed.response?.body?.usage;
+              if (usage) {
+                const contextSize = getContextSizeForModel(parsed.body?.model);
+                const cwData = buildContextWindowEvent(usage, contextSize);
+                if (cwData) {
+                  sendEventToClients(clients, 'context_window', cwData);
+                }
+              }
+            }
+          } catch (err) {
+            // Skip invalid entries
+          }
+        });
+        notifyStatsWorker(logFile);
+      }
+
+      // 检测日志文件是否已轮转到新文件
+      const currentLogFile = getLogFile();
+      if (currentLogFile !== logFile && !watchedFiles.has(currentLogFile)) {
+        // Unwatch old file to prevent watcher leak on rotation
+        unwatchFile(logFile);
+        watchedFiles.delete(logFile);
+
+        // 流式分段广播，避免全量加载 OOM
+        const endRotTotal = countLogEntries(currentLogFile);
+        clients.forEach(client => {
+          try { client.write(`event: load_start\ndata: ${JSON.stringify({ total: endRotTotal, incremental: false })}\n\n`); } catch { }
+        });
+        streamReconstructedEntries(currentLogFile, (segment) => {
+          const data = JSON.stringify(segment);
+          clients.forEach(client => {
+            try { client.write(`event: load_chunk\ndata: ${data}\n\n`); } catch { }
+          });
+        });
+        clients.forEach(client => {
+          try { client.write(`event: load_end\ndata: {}\n\n`); } catch { }
+        });
+        watchLogFile({ ...opts, logFile: currentLogFile });
+      }
+    } catch (err) {
+      // File not yet created, will retry on next poll
+    }
+  });
+}
+
+/**
+ * Start watching the current log file + install statusLine + context window.
+ * @param {object} opts
+ * @param {string} opts.logFile - current LOG_FILE
+ * @param {Array} opts.clients - SSE clients array
+ * @param {Function} opts.getCodexPid
+ * @param {Function} opts.runParallelHook
+ * @param {Function} opts.notifyStatsWorker
+ * @param {Function} opts.getLogFile
+ */
+export function startWatching(opts) {
+  const { clients, ...watchOpts } = opts;
+  watchLogFile({ ...watchOpts, clients });
+}
+
+/** Get the watchedFiles Map (for cleanup in stopViewer). */
+export function getWatchedFiles() {
+  return watchedFiles;
+}

package/lib/otel-adapter.js

@@ -0,0 +1,251 @@
+/**
+ * OpenTelemetry Adapter - 网络报文追踪模块
+ * 将拦截的 Codex 网络请求转换为 OTel Traces 并通过 OTLP 导出
+ */
+
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import resourcesPkg from '@opentelemetry/resources';
+import semConvPkg from '@opentelemetry/semantic-conventions';
+import otelApi from '@opentelemetry/api';
+
+const { Resource } = resourcesPkg;
+const { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION, SEMRESATTRS_SERVICE_INSTANCE_ID } = semConvPkg;
+const { trace, SpanKind, SpanStatusCode } = otelApi;
+
+// OTel 配置
+const OTEL_COLLECTOR_URL = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces';
+const ENABLE_OTEL = process.env.CXV_OTEL_ENABLED !== '0'; // 默认开启
+
+// 当前追踪会话ID
+let _traceSessionId = null;
+let _sdk = null;
+let _tracer = null;
+
+/**
+ * 初始化 OTel SDK
+ */
+export function initOtelAdapter(projectName, sessionId) {
+  if (!ENABLE_OTEL) return false;
+
+  _traceSessionId = sessionId || generateSessionId();
+
+  try {
+    const resource = new Resource({
+      [SEMRESATTRS_SERVICE_NAME]: 'cx-viewer',
+      [SEMRESATTRS_SERVICE_VERSION]: process.env.npm_package_version || '2.0.0',
+      [SEMRESATTRS_SERVICE_INSTANCE_ID]: _traceSessionId,
+      'cx-viewer.project': projectName || 'unknown',
+      'cx-viewer.session': _traceSessionId,
+    });
+
+    const traceExporter = new OTLPTraceExporter({
+      url: OTEL_COLLECTOR_URL,
+      headers: {},
+    });
+
+    _sdk = new NodeSDK({
+      resource,
+      traceExporter,
+      autoDetectResources: false,
+    });
+
+    _sdk.start();
+    _tracer = trace.getTracer('cx-viewer-network', '2.0.0');
+
+    process.on('SIGINT', shutdownOtel);
+    process.on('SIGTERM', shutdownOtel);
+
+    return true;
+  } catch (err) {
+    console.error('[OTel Adapter] Failed to initialize:', err.message);
+    return false;
+  }
+}
+
+/**
+ * 生成会话ID
+ */
+function generateSessionId() {
+  return `cxv-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+}
+
+/**
+ * 创建网络请求 Span
+ */
+export function createRequestSpan(requestEntry) {
+  if (!_tracer) return null;
+
+  const {
+    url,
+    method,
+    headers,
+    body,
+    timestamp,
+    project,
+    mainAgent,
+    teammate,
+    teamName,
+    isStream,
+    isHeartbeat,
+    isCountTokens,
+  } = requestEntry;
+
+  if (isHeartbeat && process.env.CXV_OTEL_SKIP_HEARTBEAT === '1') return null;
+
+  const spanName = `${method} ${getUrlPath(url)}`;
+  const startTime = timestamp ? new Date(timestamp).getTime() : Date.now();
+
+  const span = _tracer.startSpan(spanName, {
+    kind: SpanKind.CLIENT,
+    startTime,
+    attributes: {
+      'http.request.method': method,
+      'http.request.url': url,
+      'http.request.headers': JSON.stringify(sanitizeHeaders(headers)),
+      'http.request.body.size': body ? JSON.stringify(body).length : 0,
+
+      'cx.codex.main_agent': mainAgent || false,
+      'cx.codex.is_stream': isStream || false,
+      'cx.codex.is_heartbeat': isHeartbeat || false,
+      'cx.codex.is_count_tokens': isCountTokens || false,
+      'cx.codex.project': project || 'unknown',
+
+      ...(teammate && { 'cx.codex.teammate': teammate }),
+      ...(teamName && { 'cx.codex.team_name': teamName }),
+
+      ...(body?.model && { 'cx.codex.model': body.model }),
+      ...(body?.stream !== undefined && { 'cx.codex.stream_param': body.stream }),
+      ...(body?.messages && { 'cx.codex.message_count': body.messages.length }),
+      ...(body?.tools && { 'cx.codex.tool_count': body.tools.length }),
+    },
+  });
+
+  return span;
+}
+
+/**
+ * 完成 Span 并记录响应
+ */
+export function completeRequestSpan(span, requestEntry) {
+  if (!span) return;
+
+  const { response, duration } = requestEntry;
+  const endTime = Date.now();
+
+  try {
+    if (response) {
+      span.setAttribute('http.response.status_code', response.status);
+      span.setAttribute('http.response.headers', JSON.stringify(response.headers || {}));
+      span.setAttribute('http.response.body.size', response.body ? JSON.stringify(response.body).length : 0);
+
+      if (response.status >= 400) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: response.statusText || `HTTP ${response.status}`,
+        });
+      } else {
+        span.setStatus({ code: SpanStatusCode.OK });
+      }
+
+      if (response.body) {
+        const body = response.body;
+        if (body.usage) {
+          span.setAttribute('cx.codex.usage.input_tokens', body.usage.input_tokens || 0);
+          span.setAttribute('cx.codex.usage.output_tokens', body.usage.output_tokens || 0);
+          span.setAttribute('cx.codex.usage.total_tokens', body.usage.total_tokens || 0);
+        }
+        if (body.stop_reason) {
+          span.setAttribute('cx.codex.stop_reason', body.stop_reason);
+        }
+        if (body.model) {
+          span.setAttribute('cx.codex.response_model', body.model);
+        }
+      }
+    }
+
+    if (duration) {
+      span.setAttribute('http.request.duration_ms', duration);
+    }
+
+    span.end(endTime);
+  } catch (err) {
+    span.recordException(err);
+    span.end(endTime);
+  }
+}
+
+/**
+ * 记录流式响应事件
+ */
+export function recordStreamEvent(span, event) {
+  if (!span) return;
+
+  const eventType = event.type || 'unknown';
+  span.addEvent(`stream.${eventType}`, {
+    'cx.codex.event_type': eventType,
+    'cx.codex.event_data': JSON.stringify(event).slice(0, 1000),
+  });
+}
+
+/**
+ * 脱敏 Headers
+ */
+function sanitizeHeaders(headers) {
+  if (!headers) return {};
+  const safe = { ...headers };
+  const sensitive = ['authorization', 'x-api-key', 'cookie', 'x-auth-token'];
+  for (const key of Object.keys(safe)) {
+    if (sensitive.includes(key.toLowerCase())) {
+      const val = safe[key];
+      safe[key] = val?.length > 12 ? `${val.slice(0, 8)}****${val.slice(-4)}` : '****';
+    }
+  }
+  return safe;
+}
+
+/**
+ * 提取 URL path
+ */
+function getUrlPath(url) {
+  try {
+    return new URL(url).pathname;
+  } catch {
+    return url;
+  }
+}
+
+/**
+ * 关闭 OTel SDK
+ */
+export function shutdownOtel() {
+  if (_sdk) {
+    _sdk.shutdown().catch(() => {});
+    _sdk = null;
+    _tracer = null;
+  }
+}
+
+/**
+ * 获取当前会话ID
+ */
+export function getTraceSessionId() {
+  return _traceSessionId;
+}
+
+/**
+ * 检查 OTel 是否启用
+ */
+export function isOtelEnabled() {
+  return ENABLE_OTEL && _tracer !== null;
+}
+
+export default {
+  initOtelAdapter,
+  createRequestSpan,
+  completeRequestSpan,
+  recordStreamEvent,
+  shutdownOtel,
+  getTraceSessionId,
+  isOtelEnabled,
+};

package/lib/otel-receiver.js

@@ -0,0 +1,198 @@
+/**
+ * otel-receiver.js — 轻量级 OTLP HTTP 接收器
+ *
+ * 接收 Codex CLI 原生发出的 OTel trace 数据（OTLP JSON 格式），
+ * 解析 spans 并转换为 cx-viewer 的 entry 格式，写入 JSONL 日志文件。
+ *
+ * Codex 通过 config.toml [otel] 配置将遥测数据发送到此端点。
+ * 支持的数据：API 请求、工具调用、用户提示等。
+ */
+
+import { appendFileSync } from 'node:fs';
+
+/**
+ * 从 OTLP JSON trace 数据中提取 spans 并转换为 cx-viewer entries
+ * @param {object} otlpData - OTLP ExportTraceServiceRequest JSON
+ * @returns {Array} cx-viewer entry 数组
+ */
+export function parseOtlpTraces(otlpData) {
+  const entries = [];
+  if (!otlpData?.resourceSpans) return entries;
+
+  for (const rs of otlpData.resourceSpans) {
+    const resourceAttrs = extractAttributes(rs.resource?.attributes);
+
+    for (const ss of rs.scopeSpans || []) {
+      for (const span of ss.spans || []) {
+        const entry = spanToEntry(span, resourceAttrs);
+        if (entry) entries.push(entry);
+      }
+    }
+  }
+  return entries;
+}
+
+/**
+ * 将单个 OTel span 转换为 cx-viewer entry 数组
+ * 一个 span 可能包含多种事件（api_request + tool.call 等），全部提取
+ */
+function spanToEntry(span, resourceAttrs) {
+  const attrs = extractAttributes(span.attributes);
+  const events = (span.events || []).map(e => ({
+    name: e.name,
+    attrs: extractAttributes(e.attributes),
+    timeUnixNano: e.timeUnixNano,
+  }));
+
+  if (events.length === 0) return null;
+
+  const startNano = BigInt(span.startTimeUnixNano || '0');
+  const endNano = BigInt(span.endTimeUnixNano || '0');
+  const durationMs = Number((endNano - startNano) / 1000000n);
+  const timestamp = new Date(Number(startNano / 1000000n)).toISOString();
+  const model = attrs['codex.model'] || attrs['model'] || resourceAttrs['model'] || null;
+  const project = resourceAttrs['cx-viewer.project'] || resourceAttrs['service.name'] || 'codex';
+
+  const results = [];
+
+  // API 请求事件
+  for (const evt of events.filter(e => e.name === 'codex.api_request')) {
+    const a = evt.attrs;
+    results.push({
+      timestamp,
+      project,
+      url: `codex://api/${span.name || 'request'}`,
+      method: 'POST',
+      headers: {},
+      body: { model: model || a['model'], _otelSpanName: span.name },
+      response: {
+        status: parseInt(a['http.response.status_code']) || 200,
+        statusText: a['success'] === 'true' ? 'OK' : (a['error.message'] || 'Error'),
+        headers: {},
+        body: {
+          usage: {
+            input_tokens: parseInt(a['input_token_count']) || 0,
+            output_tokens: parseInt(a['output_token_count']) || 0,
+            cache_read_input_tokens: parseInt(a['cached_token_count']) || 0,
+          },
+          model: model || a['model'],
+        },
+      },
+      duration: parseInt(a['duration_ms']) || durationMs,
+      isStream: true,
+      mainAgent: true,
+      _otelSource: true,
+      _otelTraceId: span.traceId,
+      _otelSpanId: span.spanId,
+    });
+  }
+
+  // 工具调用事件
+  const toolResultEvents = events.filter(e => e.name === 'codex.tool_result');
+  for (const tc of events.filter(e => e.name === 'codex.tool.call')) {
+    const a = tc.attrs;
+    const toolName = a['tool_name'] || 'unknown';
+    const matchResult = toolResultEvents.find(tr => tr.attrs['call_id'] === a['call_id']);
+    results.push({
+      timestamp: tc.timeUnixNano
+        ? new Date(Number(BigInt(tc.timeUnixNano) / 1000000n)).toISOString()
+        : timestamp,
+      project,
+      url: `codex://tool/${toolName}`,
+      method: 'TOOL',
+      headers: {},
+      body: {
+        tool_name: toolName,
+        tool_input: safeJsonParse(a['arguments']),
+        _callId: a['call_id'],
+        _source: a['source'],
+      },
+      response: matchResult ? {
+        status: 200, statusText: 'OK', headers: {},
+        body: {
+          output: safeJsonParse(matchResult.attrs['output']) || matchResult.attrs['output'],
+          _outputLength: parseInt(matchResult.attrs['output_length']) || 0,
+        },
+      } : null,
+      duration: parseInt(a['duration_ms'] || matchResult?.attrs['duration_ms']) || 0,
+      isStream: false,
+      mainAgent: false,
+      _otelSource: true,
+      _otelTraceId: span.traceId,
+    });
+  }
+
+  // 用户提示事件
+  for (const evt of events.filter(e => e.name === 'codex.user_prompt')) {
+    const a = evt.attrs;
+    results.push({
+      timestamp: evt.timeUnixNano
+        ? new Date(Number(BigInt(evt.timeUnixNano) / 1000000n)).toISOString()
+        : timestamp,
+      project,
+      url: 'codex://user_prompt',
+      method: 'POST',
+      headers: {},
+      body: {
+        model,
+        messages: [{ role: 'user', content: a['prompt'] || '' }],
+        _promptLength: parseInt(a['prompt_length']) || 0,
+      },
+      response: null,
+      duration: 0,
+      isStream: false,
+      mainAgent: true,
+      _otelSource: true,
+      _otelTraceId: span.traceId,
+    });
+  }
+
+  return results.length > 0 ? results : null;
+}
+
+/**
+ * 提取 OTLP attributes 数组为普通对象
+ * OTLP 格式: [{ key: 'foo', value: { stringValue: 'bar' } }]
+ */
+function extractAttributes(attrs) {
+  if (!Array.isArray(attrs)) return {};
+  const result = {};
+  for (const attr of attrs) {
+    if (!attr?.key) continue;
+    const v = attr.value;
+    if (!v) continue;
+    if (v.stringValue !== undefined) result[attr.key] = v.stringValue;
+    else if (v.intValue !== undefined) result[attr.key] = String(v.intValue);
+    else if (v.doubleValue !== undefined) result[attr.key] = String(v.doubleValue);
+    else if (v.boolValue !== undefined) result[attr.key] = String(v.boolValue);
+    else if (v.arrayValue) result[attr.key] = JSON.stringify(v.arrayValue.values);
+    else if (v.kvlistValue) result[attr.key] = JSON.stringify(v.kvlistValue.values);
+  }
+  return result;
+}
+
+/**
+ * 将 OTel entries 写入 JSONL 日志文件
+ * @param {string} logFile - 日志文件路径
+ * @param {Array} entries - cx-viewer entry 数组
+ */
+export function writeOtelEntries(logFile, entries) {
+  if (!logFile || !entries.length) return;
+  const lines = entries
+    .flat() // spanToEntry 可能返回数组（多个工具调用）
+    .filter(Boolean)
+    .map(e => JSON.stringify(e) + '\n---\n')
+    .join('');
+  try {
+    appendFileSync(logFile, lines);
+  } catch (err) {
+    if (process.env.CXV_DEBUG) {
+      console.error('[OTel Receiver] Write error:', err.message);
+    }
+  }
+}
+
+function safeJsonParse(s) {
+  if (!s || typeof s !== 'string') return s;
+  try { return JSON.parse(s); } catch { return s; }
+}