@codehourra/llm-iwiki 0.1.0

package/src/collectors/codebuddy.ts

@@ -0,0 +1,164 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+
+import type { Collector, RawMessage, RawSession } from './types'
+import { clampTitle, deriveTitle, isEphemeralPath, normalizeContentParts } from './util'
+
+const APP_SUPPORT = 'Library/Application Support'
+const EXTENSION_DIRS = ['CodeBuddyExtension', 'CodeBuddy CN']
+
+interface ConversationMeta {
+  id: string
+  name?: string
+  createdAt?: string
+  lastMessageAt?: string
+}
+
+function findHistoryDirs(root: string, acc: string[], depth = 0): void {
+  if (depth > 6) return
+  let entries
+  try {
+    entries = readdirSync(root, { withFileTypes: true })
+  } catch {
+    return
+  }
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+    const full = join(root, entry.name)
+    if (entry.name === 'history') {
+      acc.push(full)
+    } else if (entry.name !== 'messages') {
+      findHistoryDirs(full, acc, depth + 1)
+    }
+  }
+}
+
+function readJson<T>(filePath: string): T | null {
+  try {
+    return JSON.parse(readFileSync(filePath, 'utf8')) as T
+  } catch {
+    return null
+  }
+}
+
+function extractMessageText(messageField: unknown): string {
+  if (typeof messageField !== 'string') return normalizeContentParts(messageField)
+  const trimmed = messageField.trim()
+  if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+    try {
+      const parsed = JSON.parse(trimmed) as Record<string, unknown>
+      if (parsed && typeof parsed === 'object' && 'content' in parsed) {
+        return normalizeContentParts((parsed as { content: unknown }).content)
+      }
+      return normalizeContentParts(parsed)
+    } catch {
+      return messageField
+    }
+  }
+  return messageField
+}
+
+function extractWorkspaceFolder(text: string): string | null {
+  const match = text.match(/Workspace Folder:\s*(.+)/)
+  if (!match) return null
+  const folder = match[1]!.trim()
+  return folder === '' ? null : folder
+}
+
+function readConversation(
+  convDir: string,
+  conversationId: string,
+  meta: ConversationMeta,
+): RawSession | null {
+  const convIndex = readJson<{ messages?: Array<{ id: string; role?: string }> }>(join(convDir, 'index.json'))
+  if (!convIndex?.messages) return null
+
+  const messages: RawMessage[] = []
+  let rawProjectPath: string | null = null
+
+  for (const entry of convIndex.messages) {
+    const raw = readJson<{ role?: string; message?: unknown }>(join(convDir, 'messages', `${entry.id}.json`))
+    if (!raw) continue
+    const role = entry.role ?? raw.role ?? 'user'
+    if (role !== 'user' && role !== 'assistant') continue
+
+    const content = extractMessageText(raw.message)
+    if (content.trim() === '') continue
+
+    if (!rawProjectPath && role === 'user') {
+      rawProjectPath = extractWorkspaceFolder(content)
+    }
+    messages.push({ role, content, timestamp: null })
+  }
+
+  if (messages.length === 0) return null
+  if (isEphemeralPath(rawProjectPath)) return null
+
+  const title = meta.name && meta.name.trim() !== '' ? clampTitle(meta.name.trim()) : deriveTitle(messages)
+
+  return {
+    sourceSessionId: conversationId,
+    rawPath: convDir,
+    rawProjectPath,
+    title,
+    createdAt: meta.createdAt ?? null,
+    updatedAt: meta.lastMessageAt ?? null,
+    messages,
+  }
+}
+
+function collectFromHistory(historyDir: string, sessions: RawSession[]): void {
+  for (const workspaceEntry of readdirSync(historyDir, { withFileTypes: true })) {
+    if (!workspaceEntry.isDirectory()) continue
+    const workspaceDir = join(historyDir, workspaceEntry.name)
+    const workspaceIndex = readJson<{ conversations?: ConversationMeta[] }>(join(workspaceDir, 'index.json'))
+    if (!workspaceIndex?.conversations?.length) continue
+
+    for (const conversation of workspaceIndex.conversations) {
+      if (!conversation.id) continue
+      const convDir = join(workspaceDir, conversation.id)
+      if (!existsSync(convDir)) continue
+      try {
+        const session = readConversation(convDir, conversation.id, conversation)
+        if (session) sessions.push(session)
+      } catch {
+        continue
+      }
+    }
+  }
+}
+
+function extensionRoots(homeDir: string): string[] {
+  return EXTENSION_DIRS.map((dir) => join(homeDir, APP_SUPPORT, dir)).filter((dir) => existsSync(dir))
+}
+
+export const codebuddyCollector: Collector = {
+  id: 'codebuddy',
+  name: 'CodeBuddy',
+
+  detect(homeDir: string): boolean {
+    return extensionRoots(homeDir).length > 0
+  },
+
+  collect(homeDir: string): RawSession[] {
+    const sessions: RawSession[] = []
+    const seen = new Set<string>()
+    for (const root of extensionRoots(homeDir)) {
+      const historyDirs: string[] = []
+      findHistoryDirs(join(root, 'Data'), historyDirs)
+      for (const historyDir of historyDirs) {
+        try {
+          if (!statSync(historyDir).isDirectory()) continue
+        } catch {
+          continue
+        }
+        collectFromHistory(historyDir, sessions)
+      }
+    }
+    return sessions.filter((session) => {
+      if (seen.has(session.sourceSessionId)) return false
+      seen.add(session.sourceSessionId)
+      return true
+    })
+  },
+}

package/src/collectors/codex.ts

@@ -0,0 +1,130 @@
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
+import { basename, join } from 'node:path'
+
+import type { Collector, RawMessage, RawSession } from './types'
+import { deriveTitle, isEphemeralPath } from './util'
+
+const SESSION_ROOTS = ['.codex/sessions', '.codex-internal/sessions']
+
+function cleanCodexText(text: string): string {
+  return text
+    .replace(/\[REQ:[0-9a-f-]+\]/gi, '')
+    .replace(/\[AMP_DONE:[0-9a-f-]+\]/gi, '')
+    .trim()
+}
+
+function walkRolloutFiles(dir: string, acc: string[]): void {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name)
+    if (entry.isDirectory()) {
+      walkRolloutFiles(full, acc)
+    } else if (entry.isFile() && entry.name.startsWith('rollout-') && entry.name.endsWith('.jsonl')) {
+      acc.push(full)
+    }
+  }
+}
+
+function parseRolloutFile(filePath: string): RawSession | null {
+  const raw = readFileSync(filePath, 'utf8')
+  const messages: RawMessage[] = []
+  let rawProjectPath: string | null = null
+  let sourceSessionId: string | null = null
+  let createdAt: string | null = null
+
+  for (const line of raw.split('\n')) {
+    const trimmed = line.trim()
+    if (trimmed === '') continue
+
+    let entry: Record<string, unknown>
+    try {
+      entry = JSON.parse(trimmed) as Record<string, unknown>
+    } catch {
+      continue
+    }
+
+    const payload = entry.payload
+    if (!payload || typeof payload !== 'object') continue
+    const payloadRecord = payload as Record<string, unknown>
+    const timestamp = typeof entry.timestamp === 'string' ? entry.timestamp : null
+
+    if (entry.type === 'session_meta') {
+      if (typeof payloadRecord.id === 'string') sourceSessionId = payloadRecord.id
+      if (typeof payloadRecord.cwd === 'string') rawProjectPath = payloadRecord.cwd
+      if (typeof payloadRecord.timestamp === 'string') createdAt = payloadRecord.timestamp
+      continue
+    }
+
+    if (entry.type !== 'event_msg') continue
+    const eventType = payloadRecord.type
+    if (eventType !== 'user_message' && eventType !== 'agent_message') continue
+    if (typeof payloadRecord.message !== 'string') continue
+
+    const content = cleanCodexText(payloadRecord.message)
+    if (content === '') continue
+    messages.push({
+      role: eventType === 'user_message' ? 'user' : 'assistant',
+      content,
+      timestamp,
+    })
+  }
+
+  if (messages.length === 0) return null
+  if (isEphemeralPath(rawProjectPath)) return null
+
+  sourceSessionId ??= basename(filePath).replace(/\.jsonl$/, '')
+  const timestamps = messages.map((message) => message.timestamp).filter((value): value is string => value != null)
+
+  return {
+    sourceSessionId,
+    rawPath: filePath,
+    rawProjectPath,
+    title: deriveTitle(messages),
+    createdAt: createdAt ?? timestamps[0] ?? null,
+    updatedAt: timestamps[timestamps.length - 1] ?? null,
+    messages,
+  }
+}
+
+function listRolloutFiles(homeDir: string): string[] {
+  const files: string[] = []
+  const seenRoots = new Set<string>()
+  for (const root of SESSION_ROOTS) {
+    const sessionsDir = join(homeDir, root)
+    if (!existsSync(sessionsDir)) continue
+
+    let canonicalRoot: string
+    try {
+      canonicalRoot = realpathSync(sessionsDir)
+    } catch {
+      canonicalRoot = sessionsDir
+    }
+    if (seenRoots.has(canonicalRoot)) continue
+    seenRoots.add(canonicalRoot)
+
+    walkRolloutFiles(sessionsDir, files)
+  }
+  return files
+}
+
+export const codexCollector: Collector = {
+  id: 'codex',
+  name: 'Codex',
+
+  detect(homeDir: string): boolean {
+    return SESSION_ROOTS.some((root) => existsSync(join(homeDir, root)))
+  },
+
+  collect(homeDir: string): RawSession[] {
+    const sessions: RawSession[] = []
+    for (const filePath of listRolloutFiles(homeDir)) {
+      try {
+        if (!statSync(filePath).isFile()) continue
+        const session = parseRolloutFile(filePath)
+        if (session) sessions.push(session)
+      } catch {
+        continue
+      }
+    }
+    return sessions
+  },
+}

package/src/collectors/cursor.ts

@@ -0,0 +1,178 @@
+import { Database } from 'bun:sqlite'
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import type { Collector, RawMessage, RawSession } from './types'
+import { clampTitle, deriveTitle, isEphemeralPath } from './util'
+
+const CURSOR_USER_DIR = 'Library/Application Support/Cursor/User'
+
+interface ComposerMeta {
+  cwd: string | null
+  name: string | null
+  createdAt: string | null
+  updatedAt: string | null
+}
+
+function epochToIso(value: unknown): string | null {
+  if (typeof value !== 'number' || !Number.isFinite(value)) return null
+  const date = new Date(value)
+  return Number.isNaN(date.getTime()) ? null : date.toISOString()
+}
+
+function folderUriToPath(uri: string): string | null {
+  if (!uri.startsWith('file://')) return null
+  try {
+    return decodeURIComponent(uri.replace(/^file:\/\//, '')).replace(/\/+$/, '')
+  } catch {
+    return null
+  }
+}
+
+function openReadonly(path: string): Database | null {
+  try {
+    return new Database(path, { readonly: true })
+  } catch {
+    return null
+  }
+}
+
+function collectWorkspaceComposers(userDir: string): Map<string, ComposerMeta> {
+  const composers = new Map<string, ComposerMeta>()
+  const workspaceStorage = join(userDir, 'workspaceStorage')
+  if (!existsSync(workspaceStorage)) return composers
+
+  for (const entry of readdirSync(workspaceStorage, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue
+    const wsDir = join(workspaceStorage, entry.name)
+
+    let cwd: string | null = null
+    const workspaceJson = join(wsDir, 'workspace.json')
+    if (existsSync(workspaceJson)) {
+      try {
+        const parsed = JSON.parse(readFileSync(workspaceJson, 'utf8')) as { folder?: string }
+        if (typeof parsed.folder === 'string') cwd = folderUriToPath(parsed.folder)
+      } catch {
+        // ignore
+      }
+    }
+
+    const dbPath = join(wsDir, 'state.vscdb')
+    if (!existsSync(dbPath)) continue
+    const db = openReadonly(dbPath)
+    if (!db) continue
+    try {
+      const row = db.query<{ value: string }, [string]>('SELECT value FROM ItemTable WHERE key = ?').get('composer.composerData')
+      if (!row) continue
+      const data = JSON.parse(row.value) as { allComposers?: Array<Record<string, unknown>> }
+      for (const composer of data.allComposers ?? []) {
+        const composerId = composer.composerId
+        if (typeof composerId !== 'string') continue
+        if (composers.has(composerId)) continue
+        composers.set(composerId, {
+          cwd,
+          name: typeof composer.name === 'string' ? composer.name : null,
+          createdAt: epochToIso(composer.createdAt),
+          updatedAt: epochToIso(composer.lastUpdatedAt),
+        })
+      }
+    } catch {
+      // ignore malformed workspace db
+    } finally {
+      db.close()
+    }
+  }
+
+  return composers
+}
+
+function buildSession(
+  globalDb: Database,
+  dbPath: string,
+  composerId: string,
+  meta: ComposerMeta,
+): RawSession | null {
+  const composerRow = globalDb
+    .query<{ value: string }, [string]>('SELECT value FROM cursorDiskKV WHERE key = ?')
+    .get(`composerData:${composerId}`)
+  if (!composerRow) return null
+
+  let composer: { name?: string; createdAt?: number; fullConversationHeadersOnly?: Array<{ bubbleId: string; type: number }> }
+  try {
+    composer = JSON.parse(composerRow.value)
+  } catch {
+    return null
+  }
+
+  const headers = composer.fullConversationHeadersOnly ?? []
+  if (headers.length === 0) return null
+
+  const bubbleStmt = globalDb.query<{ value: string }, [string]>('SELECT value FROM cursorDiskKV WHERE key = ?')
+  const messages: RawMessage[] = []
+  for (const header of headers) {
+    if (header.type !== 1 && header.type !== 2) continue
+    const bubbleRow = bubbleStmt.get(`bubbleId:${composerId}:${header.bubbleId}`)
+    if (!bubbleRow) continue
+    let bubble: { text?: string }
+    try {
+      bubble = JSON.parse(bubbleRow.value)
+    } catch {
+      continue
+    }
+    const text = typeof bubble.text === 'string' ? bubble.text.trim() : ''
+    if (text === '') continue
+    messages.push({ role: header.type === 1 ? 'user' : 'assistant', content: text, timestamp: null })
+  }
+
+  if (messages.length === 0) return null
+  if (isEphemeralPath(meta.cwd)) return null
+
+  const name = meta.name ?? composer.name ?? null
+  const title = name && name.trim() !== '' ? clampTitle(name.trim()) : deriveTitle(messages)
+
+  return {
+    sourceSessionId: composerId,
+    rawPath: `${dbPath}#${composerId}`,
+    rawProjectPath: meta.cwd,
+    title,
+    createdAt: meta.createdAt ?? epochToIso(composer.createdAt),
+    updatedAt: meta.updatedAt,
+    messages,
+  }
+}
+
+export const cursorCollector: Collector = {
+  id: 'cursor',
+  name: 'Cursor',
+
+  detect(homeDir: string): boolean {
+    return existsSync(join(homeDir, CURSOR_USER_DIR, 'globalStorage', 'state.vscdb'))
+  },
+
+  collect(homeDir: string): RawSession[] {
+    const userDir = join(homeDir, CURSOR_USER_DIR)
+    const globalDbPath = join(userDir, 'globalStorage', 'state.vscdb')
+    if (!existsSync(globalDbPath)) return []
+
+    const composers = collectWorkspaceComposers(userDir)
+    if (composers.size === 0) return []
+
+    const globalDb = openReadonly(globalDbPath)
+    if (!globalDb) return []
+
+    const sessions: RawSession[] = []
+    try {
+      for (const [composerId, meta] of composers) {
+        try {
+          const session = buildSession(globalDb, globalDbPath, composerId, meta)
+          if (session) sessions.push(session)
+        } catch {
+          continue
+        }
+      }
+    } finally {
+      globalDb.close()
+    }
+    return sessions
+  },
+}

package/src/collectors/gemini.ts

@@ -0,0 +1,141 @@
+import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
+import { basename, dirname, join } from 'node:path'
+
+import type { Collector, RawMessage, RawSession } from './types'
+import { deriveTitle, isEphemeralPath } from './util'
+
+const INTERNAL_ROOTS = ['.gemini-internal', '.gemini']
+
+function loadProjectNameMap(geminiRoot: string): Map<string, string> {
+  const nameToPath = new Map<string, string>()
+  const projectsFile = join(geminiRoot, 'projects.json')
+  if (!existsSync(projectsFile)) return nameToPath
+  try {
+    const parsed = JSON.parse(readFileSync(projectsFile, 'utf8')) as { projects?: Record<string, string> }
+    for (const [path, name] of Object.entries(parsed.projects ?? {})) {
+      if (typeof name === 'string' && !nameToPath.has(name)) nameToPath.set(name, path)
+    }
+  } catch {
+    // ignore malformed projects.json
+  }
+  return nameToPath
+}
+
+function looksLikeAbsolutePath(value: string): boolean {
+  const trimmed = value.trim()
+  return trimmed.startsWith('/') && !trimmed.includes('\n') && trimmed.length < 256
+}
+
+function parseChatFile(filePath: string, projectDir: string, nameToPath: Map<string, string>): RawSession | null {
+  let parsed: {
+    sessionId?: string
+    startTime?: string
+    lastUpdated?: string
+    messages?: Array<{ type?: string; content?: unknown; timestamp?: string }>
+  }
+  try {
+    parsed = JSON.parse(readFileSync(filePath, 'utf8'))
+  } catch {
+    return null
+  }
+
+  const messages: RawMessage[] = []
+  for (const message of parsed.messages ?? []) {
+    if (message.type !== 'user' && message.type !== 'gemini') continue
+    const content = typeof message.content === 'string' ? message.content : ''
+    if (content.trim() === '') continue
+    messages.push({
+      role: message.type === 'user' ? 'user' : 'assistant',
+      content,
+      timestamp: typeof message.timestamp === 'string' ? message.timestamp : null,
+    })
+  }
+
+  if (messages.length === 0) return null
+
+  let rawProjectPath = nameToPath.get(projectDir) ?? null
+  if (!rawProjectPath) {
+    const firstUser = messages.find((message) => message.role === 'user')
+    if (firstUser && looksLikeAbsolutePath(firstUser.content)) {
+      rawProjectPath = firstUser.content.trim().replace(/\/+$/, '')
+    }
+  }
+  if (isEphemeralPath(rawProjectPath)) return null
+
+  const timestamps = messages.map((message) => message.timestamp).filter((value): value is string => value != null)
+
+  return {
+    sourceSessionId: parsed.sessionId ?? basename(filePath).replace(/\.json$/, ''),
+    rawPath: filePath,
+    rawProjectPath,
+    title: deriveTitle(messages),
+    createdAt: parsed.startTime ?? timestamps[0] ?? null,
+    updatedAt: parsed.lastUpdated ?? timestamps[timestamps.length - 1] ?? null,
+    messages,
+  }
+}
+
+function resolveGeminiRoot(homeDir: string): string | null {
+  const seen = new Set<string>()
+  for (const root of INTERNAL_ROOTS) {
+    const dir = join(homeDir, root)
+    if (!existsSync(dir)) continue
+    let canonical: string
+    try {
+      canonical = realpathSync(dir)
+    } catch {
+      canonical = dir
+    }
+    if (seen.has(canonical)) continue
+    seen.add(canonical)
+    return dir
+  }
+  return null
+}
+
+function listChatFiles(geminiRoot: string): string[] {
+  const tmpDir = join(geminiRoot, 'tmp')
+  if (!existsSync(tmpDir)) return []
+
+  const files: string[] = []
+  for (const projectEntry of readdirSync(tmpDir, { withFileTypes: true })) {
+    if (!projectEntry.isDirectory()) continue
+    const chatsDir = join(tmpDir, projectEntry.name, 'chats')
+    if (!existsSync(chatsDir)) continue
+    for (const fileEntry of readdirSync(chatsDir, { withFileTypes: true })) {
+      if (fileEntry.isFile() && fileEntry.name.startsWith('session-') && fileEntry.name.endsWith('.json')) {
+        files.push(join(chatsDir, fileEntry.name))
+      }
+    }
+  }
+  return files
+}
+
+export const geminiCollector: Collector = {
+  id: 'gemini',
+  name: 'Gemini',
+
+  detect(homeDir: string): boolean {
+    const root = resolveGeminiRoot(homeDir)
+    return root != null && existsSync(join(root, 'tmp'))
+  },
+
+  collect(homeDir: string): RawSession[] {
+    const geminiRoot = resolveGeminiRoot(homeDir)
+    if (!geminiRoot) return []
+    const nameToPath = loadProjectNameMap(geminiRoot)
+
+    const sessions: RawSession[] = []
+    for (const filePath of listChatFiles(geminiRoot)) {
+      try {
+        if (!statSync(filePath).isFile()) continue
+        const projectDir = basename(dirname(dirname(filePath)))
+        const session = parseChatFile(filePath, projectDir, nameToPath)
+        if (session) sessions.push(session)
+      } catch {
+        continue
+      }
+    }
+    return sessions
+  },
+}

package/src/collectors/index.ts

@@ -0,0 +1,20 @@
+import { claudeCodeCollector } from './claude-code'
+import { codebuddyCollector } from './codebuddy'
+import { codexCollector } from './codex'
+import { cursorCollector } from './cursor'
+import { geminiCollector } from './gemini'
+import type { Collector } from './types'
+
+export const COLLECTORS: Collector[] = [
+  claudeCodeCollector,
+  codexCollector,
+  cursorCollector,
+  geminiCollector,
+  codebuddyCollector,
+]
+
+export function getCollector(id: string): Collector | null {
+  return COLLECTORS.find((collector) => collector.id === id) ?? null
+}
+
+export type { Collector, RawMessage, RawSession } from './types'

package/src/collectors/types.ts

@@ -0,0 +1,22 @@
+export interface RawMessage {
+  role: string
+  content: string
+  timestamp: string | null
+}
+
+export interface RawSession {
+  sourceSessionId: string
+  rawPath: string
+  rawProjectPath: string | null
+  title: string | null
+  createdAt: string | null
+  updatedAt: string | null
+  messages: RawMessage[]
+}
+
+export interface Collector {
+  id: string
+  name: string
+  detect(homeDir: string): boolean
+  collect(homeDir: string): RawSession[]
+}