lumencode 0.4.3

package/lib/parsers/codex.js

@@ -0,0 +1,316 @@
+import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
+import { join, basename, dirname } from 'path';
+import { BaseParser } from './base.js';
+import { createUsageRecord } from '../models/usage-record.js';
+
+export class CodexParser extends BaseParser {
+  getInfo() {
+    return {
+      name: 'codex',
+      displayName: 'OpenAI Codex',
+      defaultDir: '~/.codex',
+      envVar: 'CODEX_HOME',
+    };
+  }
+
+  async detect(config) {
+    const dir = this.getDataDir(config);
+    if (!dir) return false;
+    try {
+      const sessionsDir = join(dir, 'sessions');
+      return statSync(sessionsDir).isDirectory();
+    } catch {
+      try {
+        const archivedDir = join(dir, 'archived_sessions');
+        return statSync(archivedDir).isDirectory();
+      } catch {
+        return false;
+      }
+    }
+  }
+
+  async parse(config, options = {}) {
+    const dir = this.getDataDir(config);
+    const records = [];
+    if (!dir) return records;
+
+    const files = this._collectJsonlFiles(dir);
+    const parsedSessionIds = new Set();
+
+    for (const filePath of files) {
+      try {
+        const fileRecords = this._parseFile(filePath);
+        for (const r of fileRecords) {
+          if (r.sessionId) parsedSessionIds.add(r.sessionId);
+          records.push(r);
+        }
+      } catch (err) {
+        console.warn(`Codex 解析文件失败: ${filePath}`, err.message);
+      }
+    }
+
+    // 从 state DB 的 threads 表补充缺失的会话（JSONL 被清理/归档的场景）
+    const fallbackRecords = await this._parseStateDb(dir, parsedSessionIds);
+    records.push(...fallbackRecords);
+
+    return records;
+  }
+
+  _collectJsonlFiles(dir) {
+    const files = [];
+    const sessionsDir = join(dir, 'sessions');
+    try {
+      if (statSync(sessionsDir).isDirectory()) {
+        files.push(...this._walkDir(sessionsDir));
+      }
+    } catch {}
+
+    const archivedDir = join(dir, 'archived_sessions');
+    try {
+      if (statSync(archivedDir).isDirectory()) {
+        const archived = readdirSync(archivedDir).filter(f => f.endsWith('.jsonl'));
+        for (const f of archived) {
+          files.push(join(archivedDir, f));
+        }
+      }
+    } catch {}
+
+    return files;
+  }
+
+  _walkDir(dir) {
+    const results = [];
+    try {
+      const entries = readdirSync(dir);
+      for (const entry of entries) {
+        const fullPath = join(dir, entry);
+        try {
+          const stat = statSync(fullPath);
+          if (stat.isDirectory()) {
+            results.push(...this._walkDir(fullPath));
+          } else if (entry.endsWith('.jsonl')) {
+            results.push(fullPath);
+          }
+        } catch {}
+      }
+    } catch {}
+    return results;
+  }
+
+  _looksLikeNonProject(name) {
+    if (!name || name.length < 2) return true;
+    // 日期格式：2026-05-20, 20260520
+    if (/^\d{4}-\d{2}-\d{2}$/.test(name)) return true;
+    if (/^\d{8}$/.test(name)) return true;
+    // 纯数字（如 20）
+    if (/^\d+$/.test(name)) return true;
+    // Hash：16-64 位十六进制
+    if (/^[0-9a-f]{16,64}$/i.test(name)) return true;
+    return false;
+  }
+
+  _inferProject(filePath) {
+    let dir = dirname(filePath);
+    const root = dirname(dirname(filePath));
+    while (dir !== root && dir !== dirname(dir)) {
+      const dirName = basename(dir);
+      if (dirName === 'sessions' || dirName === 'archived_sessions') {
+        return '';
+      }
+      if (!this._looksLikeNonProject(dirName)) {
+        return dirName;
+      }
+      dir = dirname(dir);
+    }
+    return '';
+  }
+
+  _parseFile(filePath) {
+    const content = readFileSync(filePath, 'utf-8');
+    const lines = content.split('\n').filter(l => l.trim());
+
+    let sessionId = '';
+    let currentModel = '';
+    let lastTokenUsage = null;
+    let project = '';
+    const records = [];
+    const pendingToolCalls = [];
+    const userTexts = [];
+
+    for (const line of lines) {
+      try {
+        const event = JSON.parse(line);
+
+        if (event.type === 'session_meta' && event.payload) {
+          if (event.payload.id) sessionId = event.payload.id;
+          if (event.payload.project) {
+            project = event.payload.project;
+          } else if (event.payload.cwd) {
+            project = event.payload.cwd.replace(/\\/g, '/');
+          }
+        }
+
+        if (event.type === 'turn_context' && event.payload?.model) {
+          currentModel = event.payload.model;
+        }
+
+        // 提取用户消息文本（用于场景分类）
+        if (event.type === 'response_item' && event.payload?.role === 'user') {
+          const text = this._extractText(event.payload.content);
+          if (text && !text.startsWith('<system-reminder') && !text.startsWith('# AGENTS.md')) {
+            userTexts.push(text);
+            records.push(createUsageRecord({
+              timestamp: event.timestamp || new Date().toISOString(),
+              tool: 'codex',
+              sessionId: sessionId || basename(filePath, '.jsonl'),
+              model: '',
+              inputTokens: 0,
+              outputTokens: 0,
+              project: project || this._inferProject(filePath),
+              metadata: { type: 'user', text },
+            }));
+          }
+        }
+
+        // 收集工具调用
+        if (event.type === 'response_item' && event.payload?.type === 'function_call') {
+          pendingToolCalls.push({ name: event.payload.name || 'unknown' });
+        }
+
+        if (event.type === 'event_msg' && event.payload?.type === 'token_count') {
+          const info = event.payload.info;
+          if (!info || !info.total_token_usage) continue;
+
+          const total = info.total_token_usage;
+          const current = {
+            input: total.input_tokens || 0,
+            output: total.output_tokens || 0,
+            cachedInput: total.cached_input_tokens || 0,
+            cacheCreation: total.cache_creation_input_tokens || 0,
+            reasoningOutput: total.reasoning_output_tokens || 0,
+          };
+
+          let delta = { ...current };
+          if (lastTokenUsage) {
+            delta.input = Math.max(0, current.input - lastTokenUsage.input);
+            delta.output = Math.max(0, current.output - lastTokenUsage.output);
+            delta.cachedInput = Math.max(0, current.cachedInput - lastTokenUsage.cachedInput);
+            delta.cacheCreation = Math.max(0, current.cacheCreation - lastTokenUsage.cacheCreation);
+          }
+          lastTokenUsage = current;
+
+          if (delta.input > 0 || delta.output > 0) {
+            records.push(createUsageRecord({
+              timestamp: event.timestamp || new Date().toISOString(),
+              tool: 'codex',
+              sessionId: sessionId || basename(filePath, '.jsonl'),
+              model: currentModel || 'gpt-5',
+              inputTokens: delta.input,
+              outputTokens: delta.output,
+              cacheReadTokens: delta.cachedInput,
+              cacheWriteTokens: delta.cacheCreation,
+              costUSD: null,
+              project: project || this._inferProject(filePath),
+              metadata: {
+                type: 'assistant',
+                toolCalls: pendingToolCalls.splice(0),
+                reasoningOutputTokens: delta.reasoningOutput,
+                isFallback: !currentModel,
+              },
+            }));
+          }
+        }
+      } catch {}
+    }
+
+    return records;
+  }
+
+  _extractText(content) {
+    if (!content) return '';
+    if (typeof content === 'string') return content.trim();
+    if (Array.isArray(content)) {
+      return content
+        .filter(c => c && (c.type === 'input_text' || c.type === 'text'))
+        .map(c => c.text || '')
+        .join(' ')
+        .trim();
+    }
+    return '';
+  }
+
+  // 从 state_*.sqlite 的 threads 表提取 JSONL 已丢失的会话元数据
+  async _parseStateDb(dir, alreadyParsed) {
+    const records = [];
+    try {
+      // 查找最新版本的 state DB
+      const entries = readdirSync(dir);
+      const stateDbs = entries
+        .filter(f => /^state_\d+\.sqlite$/.test(f))
+        .sort()
+        .reverse();
+      if (stateDbs.length === 0) return records;
+
+      const dbPath = join(dir, stateDbs[0]);
+      if (!existsSync(dbPath)) return records;
+
+      const initSqlJs = (await import('sql.js')).default;
+      const SQL = await initSqlJs();
+      const dbBuf = readFileSync(dbPath);
+      const db = new SQL.Database(dbBuf);
+
+      const rows = db.exec(
+        `SELECT id, cwd, tokens_used, title, git_branch, model,
+                created_at_ms, updated_at_ms, first_user_message, archived
+         FROM threads`
+      );
+
+      if (rows[0]) {
+        for (const [sid, cwd, tokens, title, gitBranch, model, createdMs, updatedMs, firstMsg, archived] of rows[0].values) {
+          if (!sid || alreadyParsed.has(sid)) continue;
+          const project = (cwd || '').replace(/\\/g, '/');
+          const ts = createdMs ? new Date(createdMs).toISOString() : '';
+          const tsEnd = updatedMs ? new Date(updatedMs).toISOString() : ts;
+
+          // User record
+          records.push(createUsageRecord({
+            timestamp: ts,
+            tool: 'codex',
+            sessionId: sid,
+            model: '',
+            inputTokens: 0,
+            outputTokens: 0,
+            project,
+            metadata: { type: 'user', text: firstMsg || title || '', _fromStateDb: true },
+          }));
+          // Assistant record with total tokens
+          if (tokens > 0) {
+            records.push(createUsageRecord({
+              timestamp: tsEnd,
+              tool: 'codex',
+              sessionId: sid,
+              model: model || '',
+              inputTokens: tokens,
+              outputTokens: 0,
+              project,
+              metadata: { type: 'assistant', _fromStateDb: true, gitBranch },
+            }));
+          }
+        }
+      }
+      db.close();
+    } catch {}
+    return records;
+  }
+
+  async getVersion(config) {
+    const dir = this.getDataDir(config);
+    if (!dir) return null;
+    try {
+      const data = JSON.parse(readFileSync(join(dir, 'version.json'), 'utf8'));
+      return data.latest_version || null;
+    } catch {
+      return null;
+    }
+  }
+}

package/lib/parsers/index.js

@@ -0,0 +1,151 @@
+import { BaseParser } from './base.js';
+
+// 解析器注册表 - 新增工具时在此注册
+const PARSER_REGISTRY = [];
+
+/**
+ * 注册解析器类
+ * @param {typeof BaseParser} ParserClass
+ */
+export function registerParser(ParserClass) {
+  if (!(ParserClass.prototype instanceof BaseParser)) {
+    throw new Error('注册的解析器必须继承 BaseParser');
+  }
+  PARSER_REGISTRY.push(ParserClass);
+}
+
+/**
+ * 获取所有已注册的解析器实例
+ * @returns {BaseParser[]}
+ */
+export function getAllParsers() {
+  return PARSER_REGISTRY.map(P => new P());
+}
+
+/**
+ * 检测哪些工具可用（数据目录存在）
+ * @param {Object} config
+ * @returns {Promise<Array<{name, displayName, detected, dataDir}>>}
+ */
+export async function detectAvailableTools(config) {
+  const results = [];
+  for (const P of PARSER_REGISTRY) {
+    const parser = new P();
+    const info = parser.getInfo();
+    const dataDir = parser.getDataDir(config);
+    let detected = false;
+    try {
+      detected = await parser.detect(config);
+    } catch {
+      detected = false;
+    }
+    let version = null;
+    try {
+      version = await parser.getVersion(config);
+    } catch {}
+    results.push({
+      name: info.name,
+      displayName: info.displayName,
+      detected,
+      dataDir,
+      version,
+    });
+  }
+  return results;
+}
+
+/**
+ * 获取用户启用的工具列表
+ * @param {Object} config
+ * @param {Array} availableTools - detectAvailableTools 的结果
+ * @returns {Array<string>} 工具名称列表
+ */
+export function getEnabledTools(config, availableTools) {
+  if (config.enabledTools && config.enabledTools.length > 0) {
+    return config.enabledTools;
+  }
+  // 默认启用所有检测到的工具
+  return availableTools.filter(t => t.detected).map(t => t.name);
+}
+
+/**
+ * 解析指定工具的数据
+ * @param {string} toolName
+ * @param {Object} config
+ * @param {Object} options
+ * @returns {Promise<Array>}
+ */
+export async function parseTool(toolName, config, options = {}) {
+  for (const P of PARSER_REGISTRY) {
+    const parser = new P();
+    if (parser.getInfo().name === toolName) {
+      return parser.parse(config, options);
+    }
+  }
+  throw new Error(`未找到工具解析器: ${toolName}`);
+}
+
+// 统一的项目名规范化：取路径最后一段作为项目名
+function normalizeProjectToBase(projectPath) {
+  if (!projectPath) return '';
+  const normalized = projectPath.replace(/\\/g, '/').replace(/\/$/, '');
+  const parts = normalized.split('/');
+  return parts[parts.length - 1] || '';
+}
+
+/**
+ * 解析所有已启用工具的数据
+ * @param {Object} config
+ * @param {Object} options
+ * @returns {Promise<{records: Array, toolBreakdown: Object}>}
+ */
+export async function parseAllEnabledTools(config, options = {}) {
+  const available = await detectAvailableTools(config);
+  const enabled = getEnabledTools(config, available);
+  let allRecords = [];
+  const toolBreakdown = {};
+
+  for (const toolName of enabled) {
+    try {
+      const records = await parseTool(toolName, config, options);
+      allRecords.push(...records);
+      toolBreakdown[toolName] = {
+        recordCount: records.length,
+        sessionCount: new Set(records.map(r => r.sessionId)).size,
+      };
+    } catch (err) {
+      console.warn(`解析工具 ${toolName} 失败:`, err.message);
+      toolBreakdown[toolName] = { recordCount: 0, sessionCount: 0, error: err.message };
+    }
+  }
+
+  // 统一按 includeProjects 过滤
+  // Claude 记录已在解析器中通过 encoded 目录名精确匹配，直接放行
+  // Codex/OpenCode 记录通过 basename 匹配
+  if (options.includeProjects && options.includeProjects.length > 0) {
+    const allowedBases = new Set(options.includeProjects.map(p => normalizeProjectToBase(p)));
+    allRecords = allRecords.filter(r => {
+      if (r.tool === 'claude') return true;
+      const base = normalizeProjectToBase(r.project);
+      return !base || allowedBases.has(base);
+    });
+  }
+
+  // 过滤后重新计算 toolBreakdown
+  const toolGroups = {};
+  for (const r of allRecords) {
+    const t = r.tool || 'claude';
+    if (!toolGroups[t]) toolGroups[t] = { recordCount: 0, sessions: new Set() };
+    toolGroups[t].recordCount++;
+    if (r.sessionId) toolGroups[t].sessions.add(r.sessionId);
+  }
+  const filteredBreakdown = {};
+  for (const [t, g] of Object.entries(toolGroups)) {
+    filteredBreakdown[t] = { recordCount: g.recordCount, sessionCount: g.sessions.size };
+  }
+
+  // 按时间戳排序
+  allRecords.sort((a, b) => (a.timestamp || '').localeCompare(b.timestamp || ''));
+
+  return { records: allRecords, toolBreakdown: filteredBreakdown };
+}

package/lib/parsers/opencode.js

@@ -0,0 +1,216 @@
+import { existsSync, readFileSync, statSync } from 'fs';
+import { join } from 'path';
+import { BaseParser } from './base.js';
+import { createUsageRecord } from '../models/usage-record.js';
+
+// SQLite 数据库文件大小上限（500MB），超过则跳过解析避免 OOM
+const MAX_DB_SIZE = 500 * 1024 * 1024;
+
+export class OpencodeParser extends BaseParser {
+  getInfo() {
+    return {
+      name: 'opencode',
+      displayName: 'OpenCode',
+      defaultDir: '~/.local/share/opencode',
+      envVar: 'OPENCODE_DATA_DIR',
+    };
+  }
+
+  async detect(config) {
+    const dir = this.getDataDir(config);
+    if (!dir) return false;
+    try {
+      return existsSync(join(dir, 'opencode.db'));
+    } catch {
+      return false;
+    }
+  }
+
+  async parse(config, options = {}) {
+    const dir = this.getDataDir(config);
+    const records = [];
+    if (!dir) return records;
+
+    const dbPath = join(dir, 'opencode.db');
+    if (!existsSync(dbPath)) return records;
+
+    // 检查文件大小，避免大文件导致 Array buffer allocation failed
+    const fileSize = statSync(dbPath).size;
+    if (fileSize > MAX_DB_SIZE) {
+      console.warn(`OpenCode: opencode.db 过大 (${(fileSize / 1024 / 1024).toFixed(0)}MB)，跳过解析`);
+      return records;
+    }
+
+    try {
+      const initSqlJs = (await import('sql.js')).default;
+      const SQL = await initSqlJs();
+      const dbBuf = readFileSync(dbPath);
+      const db = new SQL.Database(dbBuf);
+
+      // 读取 session -> project 映射
+      const sessionMap = {};
+      // 不同版本 schema 不同，探测可用列
+      let sessCols;
+      try {
+        const colInfo = db.exec("PRAGMA table_info(session)");
+        sessCols = colInfo[0] ? colInfo[0].values.map(r => r[1]) : [];
+      } catch { sessCols = []; }
+      const hasPath = sessCols.includes('path');
+      const hasDir = sessCols.includes('directory');
+      const sessSelect = hasDir
+        ? `SELECT id, directory${hasPath ? ', path' : ''} FROM session`
+        : 'SELECT id FROM session';
+      try {
+        const sessRows = db.exec(sessSelect);
+        if (sessRows[0]) {
+          for (const row of sessRows[0].values) {
+            const id = row[0];
+            const dir = hasDir ? (row[1] || '') : '';
+            const p = hasPath ? (row[2] || '') : '';
+            sessionMap[id] = (p || dir || '').replace(/\\/g, '/');
+          }
+        }
+      } catch {}
+
+      // 读取所有 message
+      const msgRows = db.exec(
+        "SELECT id, session_id, time_created, data FROM message ORDER BY time_created"
+      );
+      if (!msgRows[0]) { db.close(); return records; }
+
+      // 计算 delta tokens（message 中 tokens 是累计值）
+      let lastTokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+
+      for (const [msgId, sessionId, timeCreated, dataStr] of msgRows[0].values) {
+        let data;
+        try { data = JSON.parse(dataStr); } catch { continue; }
+
+        const role = data.role || '';
+        const timestamp = new Date(timeCreated).toISOString();
+        const project = sessionMap[sessionId] || '';
+        const model = data.modelID || data.model?.modelID || '';
+
+        // User messages for scenario classification
+        if (role === 'user') {
+          const text = this._extractUserText(db, msgId);
+          if (text) {
+            records.push(createUsageRecord({
+              timestamp,
+              tool: 'opencode',
+              sessionId: sessionId || '',
+              model: '',
+              inputTokens: 0,
+              outputTokens: 0,
+              project,
+              metadata: { type: 'user', text },
+            }));
+          }
+          continue;
+        }
+
+        // Assistant messages with token usage
+        if (role === 'assistant' && data.tokens) {
+          const t = data.tokens;
+          const current = {
+            input: t.input || 0,
+            output: t.output || 0,
+            cacheRead: t.cache?.read || 0,
+            cacheWrite: t.cache?.write || 0,
+          };
+
+          const delta = {
+            input: Math.max(0, current.input - lastTokens.input),
+            output: Math.max(0, current.output - lastTokens.output),
+            cacheRead: Math.max(0, current.cacheRead - lastTokens.cacheRead),
+            cacheWrite: Math.max(0, current.cacheWrite - lastTokens.cacheWrite),
+          };
+          lastTokens = current;
+
+          // Collect tool calls from parts
+          const toolCalls = this._extractToolCalls(db, msgId);
+
+          if (delta.input > 0 || delta.output > 0) {
+            records.push(createUsageRecord({
+              timestamp,
+              tool: 'opencode',
+              sessionId: sessionId || '',
+              model,
+              inputTokens: delta.input,
+              outputTokens: delta.output,
+              cacheReadTokens: delta.cacheRead,
+              cacheWriteTokens: delta.cacheWrite,
+              costUSD: data.cost ?? null,
+              project,
+              metadata: {
+                type: 'assistant',
+                toolCalls,
+                reasoningOutputTokens: t.reasoning || 0,
+              },
+            }));
+          }
+        }
+      }
+
+      db.close();
+    } catch (err) {
+      console.warn('OpenCode parse error:', err.message);
+    }
+
+    return records;
+  }
+
+  _extractUserText(db, msgId) {
+    try {
+      const rows = db.exec(
+        "SELECT data FROM part WHERE message_id = ? AND json_extract(data, '$.type') = 'text'",
+        [msgId]
+      );
+      if (rows[0]?.values?.length) {
+        const parts = [];
+        for (const [dataStr] of rows[0].values) {
+          const d = JSON.parse(dataStr);
+          if (d.text) parts.push(d.text);
+        }
+        return parts.join(' ').trim();
+      }
+    } catch {}
+    return '';
+  }
+
+  _extractToolCalls(db, msgId) {
+    const calls = [];
+    try {
+      const rows = db.exec(
+        "SELECT data FROM part WHERE message_id = ? AND json_extract(data, '$.type') = 'tool'",
+        [msgId]
+      );
+      if (rows[0]?.values) {
+        for (const [dataStr] of rows[0].values) {
+          const d = JSON.parse(dataStr);
+          const name = d.name || d.tool || 'unknown';
+          calls.push({ name });
+        }
+      }
+    } catch {}
+    return calls;
+  }
+
+  async getVersion(config) {
+    const dir = this.getDataDir(config);
+    if (!dir) return null;
+    const dbPath = join(dir, 'opencode.db');
+    if (!existsSync(dbPath)) return null;
+    try {
+      if (statSync(dbPath).size > MAX_DB_SIZE) return null;
+      const initSqlJs = (await import('sql.js')).default;
+      const SQL = await initSqlJs();
+      const dbBuf = readFileSync(dbPath);
+      const db = new SQL.Database(dbBuf);
+      const rows = db.exec("SELECT value FROM settings WHERE key = 'version'");
+      db.close();
+      return rows[0]?.values?.[0]?.[0] || null;
+    } catch {
+      return null;
+    }
+  }
+}