@cloudcli-ai/cloudcli 1.30.0 → 1.31.0

package/server/modules/providers/list/codex/codex-session-synchronizer.provider.ts

@@ -0,0 +1,119 @@
+import os from 'node:os';
+import path from 'node:path';
+
+import { sessionsDb } from '@/modules/database/index.js';
+import {
+  buildLookupMap,
+  extractFirstValidJsonlData,
+  findFilesRecursivelyCreatedAfter,
+  normalizeSessionName,
+  readFileTimestamps,
+} from '@/shared/utils.js';
+import type { IProviderSessionSynchronizer } from '@/shared/interfaces.js';
+
+type ParsedSession = {
+  sessionId: string;
+  projectPath: string;
+  sessionName?: string;
+};
+
+/**
+ * Session indexer for Codex transcript artifacts.
+ */
+export class CodexSessionSynchronizer implements IProviderSessionSynchronizer {
+  private readonly provider = 'codex' as const;
+  private readonly codexHome = path.join(os.homedir(), '.codex');
+
+  /**
+   * Scans ~/.codex/sessions and upserts discovered sessions into DB.
+   */
+  async synchronize(since?: Date): Promise<number> {
+    const nameMap = await buildLookupMap(path.join(this.codexHome, 'session_index.jsonl'), 'id', 'thread_name');
+    const files = await findFilesRecursivelyCreatedAfter(
+      path.join(this.codexHome, 'sessions'),
+      '.jsonl',
+      since ?? null
+    );
+
+    let processed = 0;
+    for (const filePath of files) {
+      const parsed = await this.processSessionFile(filePath, nameMap);
+      if (!parsed) {
+        continue;
+      }
+
+      const existingSession = sessionsDb.getSessionById(parsed.sessionId);
+      if (existingSession) {
+        // If session name is untitled and we now have a name, update it
+        if (existingSession.custom_name === 'Untitled Codex Session' && parsed.sessionName && parsed.sessionName !== 'Untitled Codex Session') {
+          sessionsDb.updateSessionCustomName(parsed.sessionId, parsed.sessionName);
+        }
+      }
+
+      const timestamps = await readFileTimestamps(filePath);
+      sessionsDb.createSession(
+        parsed.sessionId,
+        this.provider,
+        parsed.projectPath,
+        parsed.sessionName,
+        timestamps.createdAt,
+        timestamps.updatedAt,
+        filePath
+      );
+      processed += 1;
+    }
+
+    return processed;
+  }
+
+  /**
+   * Parses and upserts one Codex session JSONL file.
+   */
+  async synchronizeFile(filePath: string): Promise<string | null> {
+    if (!filePath.endsWith('.jsonl')) {
+      return null;
+    }
+
+    const nameMap = await buildLookupMap(path.join(this.codexHome, 'session_index.jsonl'), 'id', 'thread_name');
+    const parsed = await this.processSessionFile(filePath, nameMap);
+    if (!parsed) {
+      return null;
+    }
+
+    const timestamps = await readFileTimestamps(filePath);
+    return sessionsDb.createSession(
+      parsed.sessionId,
+      this.provider,
+      parsed.projectPath,
+      parsed.sessionName,
+      timestamps.createdAt,
+      timestamps.updatedAt,
+      filePath
+    );
+  }
+
+  /**
+   * Extracts session metadata from one Codex JSONL session file.
+   */
+  private async processSessionFile(
+    filePath: string,
+    nameMap: Map<string, string>
+  ): Promise<ParsedSession | null> {
+    return extractFirstValidJsonlData(filePath, (rawData) => {
+      const data = rawData as Record<string, unknown>;
+      const payload = data.payload as Record<string, unknown> | undefined;
+      const sessionId = typeof payload?.id === 'string' ? payload.id : undefined;
+      const projectPath = typeof payload?.cwd === 'string' ? payload.cwd : undefined;
+
+      if (!sessionId || !projectPath) {
+        return null;
+      }
+
+      return {
+        sessionId,
+        projectPath,
+        sessionName: normalizeSessionName(nameMap.get(sessionId), 'Untitled Codex Session'),
+      };
+    });
+  }
+}

package/server/modules/providers/list/codex/codex-sessions.provider.ts

@@ -1,4 +1,7 @@
-import { getCodexSessionMessages } from '@/projects.js';
+import fsSync from 'node:fs';
+import readline from 'node:readline';
+
+import { sessionsDb } from '@/modules/database/index.js';
 import type { IProviderSessions } from '@/shared/interfaces.js';
 import type { AnyRecord, FetchHistoryOptions, FetchHistoryResult, NormalizedMessage } from '@/shared/types.js';
 import { createNormalizedMessage, generateMessageId, readObjectRecord } from '@/shared/utils.js';
@@ -11,14 +14,250 @@ type CodexHistoryResult =
       messages?: AnyRecord[];
       total?: number;
       hasMore?: boolean;
+      offset?: number;
+      limit?: number | null;
       tokenUsage?: unknown;
     };
 
-const loadCodexSessionMessages = getCodexSessionMessages as unknown as (
+function isVisibleCodexUserMessage(payload: AnyRecord | null | undefined): boolean {
+  if (!payload || payload.type !== 'user_message') {
+    return false;
+  }
+
+  if (payload.kind && payload.kind !== 'plain') {
+    return false;
+  }
+
+  return typeof payload.message === 'string' && payload.message.trim().length > 0;
+}
+
+function extractCodexTextContent(content: unknown): string {
+  if (!Array.isArray(content)) {
+    return typeof content === 'string' ? content : '';
+  }
+
+  return content
+    .map((item) => {
+      if (!item || typeof item !== 'object') {
+        return '';
+      }
+
+      const record = item as AnyRecord;
+      if (
+        (record.type === 'input_text' || record.type === 'output_text' || record.type === 'text')
+        && typeof record.text === 'string'
+      ) {
+        return record.text;
+      }
+
+      return '';
+    })
+    .filter(Boolean)
+    .join('\n');
+}
+
+async function getCodexSessionMessages(
   sessionId: string,
-  limit: number | null,
-  offset: number,
-) => Promise<CodexHistoryResult>;
+  limit: number | null = null,
+  offset = 0,
+): Promise<CodexHistoryResult> {
+  try {
+    const sessionFilePath = sessionsDb.getSessionById(sessionId)?.jsonl_path;
+
+    if (!sessionFilePath) {
+      console.warn(`Codex session file not found for session ${sessionId}`);
+      return { messages: [], total: 0, hasMore: false };
+    }
+
+    const messages: AnyRecord[] = [];
+    let tokenUsage: AnyRecord | null = null;
+    const fileStream = fsSync.createReadStream(sessionFilePath);
+    const rl = readline.createInterface({
+      input: fileStream,
+      crlfDelay: Infinity,
+    });
+
+    for await (const line of rl) {
+      if (!line.trim()) {
+        continue;
+      }
+
+      try {
+        const entry = JSON.parse(line) as AnyRecord;
+
+        if (entry.type === 'event_msg' && entry.payload?.type === 'token_count' && entry.payload?.info) {
+          const info = entry.payload.info as AnyRecord;
+          if (info.total_token_usage) {
+            const usage = info.total_token_usage as AnyRecord;
+            tokenUsage = {
+              used: usage.total_tokens || 0,
+              total: info.model_context_window || 200000,
+            };
+          }
+        }
+
+        if (entry.type === 'event_msg' && isVisibleCodexUserMessage(entry.payload as AnyRecord)) {
+          messages.push({
+            type: 'user',
+            timestamp: entry.timestamp,
+            message: {
+              role: 'user',
+              content: entry.payload.message,
+            },
+          });
+        }
+
+        if (
+          entry.type === 'response_item' &&
+          entry.payload?.type === 'message' &&
+          entry.payload.role === 'assistant'
+        ) {
+          const textContent = extractCodexTextContent(entry.payload.content);
+          if (textContent.trim()) {
+            messages.push({
+              type: 'assistant',
+              timestamp: entry.timestamp,
+              message: {
+                role: 'assistant',
+                content: textContent,
+              },
+            });
+          }
+        }
+
+        if (entry.type === 'response_item' && entry.payload?.type === 'reasoning') {
+          const summaryText = Array.isArray(entry.payload.summary)
+            ? entry.payload.summary
+                .map((item: AnyRecord) => item?.text)
+                .filter(Boolean)
+                .join('\n')
+            : '';
+
+          if (summaryText.trim()) {
+            messages.push({
+              type: 'thinking',
+              timestamp: entry.timestamp,
+              message: {
+                role: 'assistant',
+                content: summaryText,
+              },
+            });
+          }
+        }
+
+        if (entry.type === 'response_item' && entry.payload?.type === 'function_call') {
+          let toolName = entry.payload.name;
+          let toolInput = entry.payload.arguments;
+
+          if (toolName === 'shell_command') {
+            toolName = 'Bash';
+            try {
+              const args = JSON.parse(entry.payload.arguments) as AnyRecord;
+              toolInput = JSON.stringify({ command: args.command });
+            } catch {
+              // Keep original arguments when parsing fails.
+            }
+          }
+
+          messages.push({
+            type: 'tool_use',
+            timestamp: entry.timestamp,
+            toolName,
+            toolInput,
+            toolCallId: entry.payload.call_id,
+          });
+        }
+
+        if (entry.type === 'response_item' && entry.payload?.type === 'function_call_output') {
+          messages.push({
+            type: 'tool_result',
+            timestamp: entry.timestamp,
+            toolCallId: entry.payload.call_id,
+            output: entry.payload.output,
+          });
+        }
+
+        if (entry.type === 'response_item' && entry.payload?.type === 'custom_tool_call') {
+          const toolName = entry.payload.name || 'custom_tool';
+          const input = entry.payload.input || '';
+
+          if (toolName === 'apply_patch') {
+            const fileMatch = String(input).match(/\*\*\* Update File: (.+)/);
+            const filePath = fileMatch ? fileMatch[1].trim() : 'unknown';
+            const lines = String(input).split('\n');
+            const oldLines: string[] = [];
+            const newLines: string[] = [];
+
+            for (const lineContent of lines) {
+              if (lineContent.startsWith('-') && !lineContent.startsWith('---')) {
+                oldLines.push(lineContent.slice(1));
+              } else if (lineContent.startsWith('+') && !lineContent.startsWith('+++')) {
+                newLines.push(lineContent.slice(1));
+              }
+            }
+
+            messages.push({
+              type: 'tool_use',
+              timestamp: entry.timestamp,
+              toolName: 'Edit',
+              toolInput: JSON.stringify({
+                file_path: filePath,
+                old_string: oldLines.join('\n'),
+                new_string: newLines.join('\n'),
+              }),
+              toolCallId: entry.payload.call_id,
+            });
+          } else {
+            messages.push({
+              type: 'tool_use',
+              timestamp: entry.timestamp,
+              toolName,
+              toolInput: input,
+              toolCallId: entry.payload.call_id,
+            });
+          }
+        }
+
+        if (entry.type === 'response_item' && entry.payload?.type === 'custom_tool_call_output') {
+          messages.push({
+            type: 'tool_result',
+            timestamp: entry.timestamp,
+            toolCallId: entry.payload.call_id,
+            output: entry.payload.output || '',
+          });
+        }
+      } catch {
+        // Skip malformed lines.
+      }
+    }
+
+    messages.sort(
+      (a, b) => new Date(a.timestamp || 0).getTime() - new Date(b.timestamp || 0).getTime(),
+    );
+    const total = messages.length;
+
+    if (limit !== null) {
+      const startIndex = Math.max(0, total - offset - limit);
+      const endIndex = total - offset;
+      const paginatedMessages = messages.slice(startIndex, endIndex);
+      const hasMore = startIndex > 0;
+
+      return {
+        messages: paginatedMessages,
+        total,
+        hasMore,
+        offset,
+        limit,
+        tokenUsage,
+      };
+    }
+
+    return { messages, tokenUsage };
+  } catch (error) {
+    console.error(`Error reading Codex session messages for ${sessionId}:`, error);
+    return { messages: [], total: 0, hasMore: false };
+  }
+}
 
 export class CodexSessionsProvider implements IProviderSessions {
   /**
@@ -31,6 +270,23 @@ export class CodexSessionsProvider implements IProviderSessions {
     const ts = raw.timestamp || new Date().toISOString();
     const baseId = raw.uuid || generateMessageId('codex');
 
+    if (raw.type === 'thinking' || raw.isReasoning) {
+      const thinkingContent = typeof raw.message?.content === 'string'
+        ? raw.message.content
+        : '';
+      if (!thinkingContent.trim()) {
+        return [];
+      }
+      return [createNormalizedMessage({
+        id: baseId,
+        sessionId,
+        timestamp: ts,
+        provider: PROVIDER,
+        kind: 'thinking',
+        content: thinkingContent,
+      })];
+    }
+
     if (raw.message?.role === 'user') {
       const content = typeof raw.message.content === 'string'
         ? raw.message.content
@@ -77,17 +333,6 @@ export class CodexSessionsProvider implements IProviderSessions {
       })];
     }
 
-    if (raw.type === 'thinking' || raw.isReasoning) {
-      return [createNormalizedMessage({
-        id: baseId,
-        sessionId,
-        timestamp: ts,
-        provider: PROVIDER,
-        kind: 'thinking',
-        content: raw.message?.content || '',
-      })];
-    }
-
     if (raw.type === 'tool_use' || raw.toolName) {
       return [createNormalizedMessage({
         id: baseId,
@@ -275,7 +520,7 @@ export class CodexSessionsProvider implements IProviderSessions {
 
     let result: CodexHistoryResult;
     try {
-      result = await loadCodexSessionMessages(sessionId, limit, offset);
+      result = await getCodexSessionMessages(sessionId, limit, offset);
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       console.warn(`[CodexProvider] Failed to load session ${sessionId}:`, message);

package/server/modules/providers/list/codex/codex.provider.ts

@@ -1,13 +1,15 @@
 import { AbstractProvider } from '@/modules/providers/shared/base/abstract.provider.js';
 import { CodexProviderAuth } from '@/modules/providers/list/codex/codex-auth.provider.js';
 import { CodexMcpProvider } from '@/modules/providers/list/codex/codex-mcp.provider.js';
+import { CodexSessionSynchronizer } from '@/modules/providers/list/codex/codex-session-synchronizer.provider.js';
 import { CodexSessionsProvider } from '@/modules/providers/list/codex/codex-sessions.provider.js';
-import type { IProviderAuth, IProviderSessions } from '@/shared/interfaces.js';
+import type { IProviderAuth, IProviderSessionSynchronizer, IProviderSessions } from '@/shared/interfaces.js';
 
 export class CodexProvider extends AbstractProvider {
   readonly mcp = new CodexMcpProvider();
   readonly auth: IProviderAuth = new CodexProviderAuth();
   readonly sessions: IProviderSessions = new CodexSessionsProvider();
+  readonly sessionSynchronizer: IProviderSessionSynchronizer = new CodexSessionSynchronizer();
 
   constructor() {
     super('codex');

package/server/modules/providers/list/cursor/cursor-session-synchronizer.provider.ts

@@ -0,0 +1,176 @@
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import readline from 'node:readline';
+
+import { sessionsDb } from '@/modules/database/index.js';
+import {
+  extractFirstValidJsonlData,
+  findFilesRecursivelyCreatedAfter,
+  normalizeSessionName,
+  readFileTimestamps,
+} from '@/shared/utils.js';
+import type { IProviderSessionSynchronizer } from '@/shared/interfaces.js';
+
+type ParsedSession = {
+  sessionId: string;
+  projectPath: string;
+  sessionName?: string;
+};
+
+/**
+ * Returns directory entries or an empty list when the folder is missing.
+ */
+async function listDirectoryEntriesSafe(
+  directoryPath: string
+): Promise<import('node:fs').Dirent[]> {
+  try {
+    return await fsp.readdir(directoryPath, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Session indexer for Cursor transcript artifacts.
+ */
+export class CursorSessionSynchronizer implements IProviderSessionSynchronizer {
+  private readonly provider = 'cursor' as const;
+  private readonly cursorHome = path.join(os.homedir(), '.cursor');
+
+  /**
+   * Scans Cursor chats and upserts discovered sessions into DB.
+   */
+  async synchronize(since?: Date): Promise<number> {
+    const projectsDir = path.join(this.cursorHome, 'projects');
+    const projectEntries = await listDirectoryEntriesSafe(projectsDir);
+    const seenProjectPaths = new Set<string>();
+
+    let processed = 0;
+    for (const entry of projectEntries) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+
+      const workerLogPath = path.join(projectsDir, entry.name, 'worker.log');
+      const projectPath = await this.extractProjectPathFromWorkerLog(workerLogPath);
+      if (!projectPath || seenProjectPaths.has(projectPath)) {
+        continue;
+      }
+
+      seenProjectPaths.add(projectPath);
+      const projectHash = this.md5(projectPath);
+      const chatsDir = path.join(this.cursorHome, 'chats', projectHash);
+      const files = await findFilesRecursivelyCreatedAfter(chatsDir, '.jsonl', since ?? null);
+
+      for (const filePath of files) {
+        const parsed = await this.processSessionFile(filePath);
+        if (!parsed) {
+          continue;
+        }
+
+        const timestamps = await readFileTimestamps(filePath);
+        sessionsDb.createSession(
+          parsed.sessionId,
+          this.provider,
+          parsed.projectPath,
+          parsed.sessionName,
+          timestamps.createdAt,
+          timestamps.updatedAt,
+          filePath
+        );
+        processed += 1;
+      }
+    }
+
+    return processed;
+  }
+
+  /**
+   * Parses and upserts one Cursor session JSONL file.
+   */
+  async synchronizeFile(filePath: string): Promise<string | null> {
+    if (!filePath.endsWith('.jsonl')) {
+      return null;
+    }
+
+    const parsed = await this.processSessionFile(filePath);
+    if (!parsed) {
+      return null;
+    }
+
+    const timestamps = await readFileTimestamps(filePath);
+    return sessionsDb.createSession(
+      parsed.sessionId,
+      this.provider,
+      parsed.projectPath,
+      parsed.sessionName,
+      timestamps.createdAt,
+      timestamps.updatedAt,
+      filePath
+    );
+  }
+
+  /**
+   * Produces the same project hash Cursor uses in chat directory names.
+   */
+  private md5(input: string): string {
+    return crypto.createHash('md5').update(input).digest('hex');
+  }
+
+  /**
+   * Extracts project path from Cursor worker.log.
+   */
+  private async extractProjectPathFromWorkerLog(filePath: string): Promise<string | null> {
+    try {
+      const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
+      const lineReader = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+
+      for await (const line of lineReader) {
+        const match = line.match(/workspacePath=(.*)$/);
+        const projectPath = match?.[1]?.trim();
+        if (projectPath) {
+          lineReader.close();
+          fileStream.close();
+          return projectPath;
+        }
+      }
+    } catch {
+      // Missing worker logs are valid for partial or incomplete session data.
+    }
+
+    return null;
+  }
+
+  /**
+   * Extracts session metadata from one Cursor JSONL session file.
+   */
+  private async processSessionFile(filePath: string): Promise<ParsedSession | null> {
+    const sessionId = path.basename(filePath, '.jsonl');
+    const grandparentDir = path.dirname(path.dirname(filePath));
+    const workerLogPath = path.join(grandparentDir, 'worker.log');
+    const projectPath = await this.extractProjectPathFromWorkerLog(workerLogPath);
+
+    if (!projectPath) {
+      return null;
+    }
+
+    return extractFirstValidJsonlData(filePath, (rawData) => {
+      const data = rawData as Record<string, any>;
+      if (data.role !== 'user') {
+        return null;
+      }
+
+      const text = typeof data.message?.content?.[0]?.text === 'string' ? data.message.content[0].text : '';
+      const firstLine = text.replace(/<\/?user_query>/g, '').trim().split('\n')[0];
+
+      return {
+        sessionId,
+        projectPath,
+        sessionName: normalizeSessionName(firstLine, 'Untitled Cursor Session'),
+      };
+    });
+  }
+}

package/server/modules/providers/list/cursor/cursor.provider.ts

@@ -1,13 +1,15 @@
 import { AbstractProvider } from '@/modules/providers/shared/base/abstract.provider.js';
 import { CursorProviderAuth } from '@/modules/providers/list/cursor/cursor-auth.provider.js';
 import { CursorMcpProvider } from '@/modules/providers/list/cursor/cursor-mcp.provider.js';
+import { CursorSessionSynchronizer } from '@/modules/providers/list/cursor/cursor-session-synchronizer.provider.js';
 import { CursorSessionsProvider } from '@/modules/providers/list/cursor/cursor-sessions.provider.js';
-import type { IProviderAuth, IProviderSessions } from '@/shared/interfaces.js';
+import type { IProviderAuth, IProviderSessionSynchronizer, IProviderSessions } from '@/shared/interfaces.js';
 
 export class CursorProvider extends AbstractProvider {
   readonly mcp = new CursorMcpProvider();
   readonly auth: IProviderAuth = new CursorProviderAuth();
   readonly sessions: IProviderSessions = new CursorSessionsProvider();
+  readonly sessionSynchronizer: IProviderSessionSynchronizer = new CursorSessionSynchronizer();
 
   constructor() {
     super('cursor');