@dashclaw/cli 0.2.0 → 0.3.2

package/lib/code/codex-parser.vendored.js

@@ -0,0 +1,360 @@
+// VENDORED from app/lib/codex/parser.ts — GENERATED, do not edit by hand.
+// The CLI runs as raw Node ESM and is published without app/, so it cannot import
+// the app TypeScript source. Regenerate by compiling app/lib/codex/parser.ts.
+// See cli/lib/code/vendored.js for the same sibling-package rationale.
+
+// app/lib/codex/parser.js
+//
+// Codex CLI session JSONL parser.
+//
+// Codex writes session rollouts to `~/.codex/sessions/rollout-<ts>-<uuid>.jsonl`.
+// Each line has the shape:
+//
+//   { "timestamp": "...", "type": "session_meta"|"response_item"|...,
+//     "payload": { ... } }
+//
+// We normalize the rollout into a session object that's intentionally
+// parallel to the Claude Code parser output (app/lib/claude-code/parser.js)
+// so downstream insights, alerts, and rules engines can be reused with
+// minimal branching. The shape differs only where Codex's data model
+// fundamentally diverges from Claude's (token semantics, reasoning tokens).
+//
+// Token mapping (Codex → normalized):
+//   input_tokens            → input_tokens
+//   output_tokens           → output_tokens (excludes reasoning)
+//   cached_input_tokens     → cache_read_tokens
+//   reasoning_output_tokens → reasoning_tokens (Codex-only field)
+//   total_tokens            → recomputed at the request level for safety
+//
+// There is no Codex equivalent to Claude's `cache_creation_tokens`. We emit
+// 0 for that field on Codex sessions to keep the schema compatible.
+//
+// This parser is deliberately tolerant: malformed lines are counted in
+// `skippedLines` and never throw. It returns a partial session even when
+// the rollout was truncated mid-write.
+import fs from 'node:fs';
+import readline from 'node:readline';
+export const CODEX_PARSER_VERSION = 1;
+function safeParse(line) {
+    try {
+        return JSON.parse(line);
+    }
+    catch {
+        return null;
+    }
+}
+function previewText(value, max = 200) {
+    if (typeof value !== 'string')
+        return '';
+    return value.slice(0, max);
+}
+// Best-effort agent-kind tag for downstream branching. Always 'codex' here.
+export const CODEX_AGENT_KIND = 'codex';
+function _initState({ sourceFile, sourceMtime }) {
+    return {
+        session: {
+            agentKind: CODEX_AGENT_KIND,
+            sourceFile: sourceFile || null,
+            sourceMtime: sourceMtime || null,
+            sessionUuid: null,
+            projectSlug: null,
+            cwd: null,
+            gitBranch: null,
+            gitCommit: null,
+            gitRepoUrl: null,
+            cliVersion: null,
+            originator: null,
+            modelProvider: null,
+            modelPrimary: null,
+            startedAt: null,
+            endedAt: null,
+            parserVersion: CODEX_PARSER_VERSION,
+            // counters
+            jsonlRecords: 0,
+            responseItems: 0,
+            eventMessages: 0,
+            compactedItems: 0,
+            skippedLines: 0,
+            turnCount: 0,
+            toolCallCount: 0,
+            // token totals
+            totals: {
+                input_tokens: 0,
+                output_tokens: 0,
+                cache_read_tokens: 0,
+                cache_creation_tokens: 0, // always 0 for Codex; kept for schema parity
+                reasoning_tokens: 0,
+            },
+            // per-turn token snapshots, in arrival order
+            turns: [], // { turnId, inputTokens, outputTokens, cachedInputTokens, reasoningTokens, total, at }
+            // canonical message timeline (user + assistant + reasoning + tool_call + tool_result)
+            messages: [],
+            // tool uses across the session
+            toolUses: [],
+        },
+        seenTurnIds: new Set(),
+    };
+}
+function _handleSessionMeta(state, payload, ts) {
+    const s = state.session;
+    if (!s.sessionUuid && payload?.id)
+        s.sessionUuid = payload.id;
+    if (!s.cwd && payload?.cwd)
+        s.cwd = payload.cwd;
+    if (!s.cliVersion && payload?.cli_version)
+        s.cliVersion = payload.cli_version;
+    if (!s.originator && payload?.originator)
+        s.originator = payload.originator;
+    if (!s.modelProvider && payload?.model_provider)
+        s.modelProvider = payload.model_provider;
+    if (!s.startedAt)
+        s.startedAt = ts || payload?.timestamp || null;
+    if (payload?.git && typeof payload.git === 'object') {
+        const git = payload.git;
+        if (!s.gitBranch && git.branch)
+            s.gitBranch = git.branch;
+        if (!s.gitCommit && git.commit_hash)
+            s.gitCommit = git.commit_hash;
+        if (!s.gitRepoUrl && git.repository_url)
+            s.gitRepoUrl = git.repository_url;
+    }
+}
+function _handleResponseItem(state, payload, ts) {
+    const s = state.session;
+    s.responseItems += 1;
+    if (!payload || typeof payload !== 'object')
+        return;
+    const role = payload.role;
+    // Response items can be of several shapes:
+    //   - { role: 'user'|'assistant'|'system', content: [...] }
+    //   - { type: 'function_call', name, arguments, call_id }
+    //   - { type: 'function_call_output', call_id, output }
+    //   - { type: 'reasoning', summary: [...] }
+    if (payload.type === 'function_call' || (payload.name && payload.call_id)) {
+        s.toolCallCount += 1;
+        s.toolUses.push({
+            name: payload.name || 'unknown',
+            tool_use_id: payload.call_id || payload.id || null,
+            arguments_preview: previewText(payload.arguments, 240),
+            target: safeToolTarget(payload),
+            at: ts,
+        });
+        s.messages.push({
+            role: 'tool_call',
+            name: payload.name,
+            tool_use_id: payload.call_id || null,
+            preview: previewText(payload.arguments, 240),
+            at: ts,
+        });
+        return;
+    }
+    if (payload.type === 'function_call_output') {
+        s.messages.push({
+            role: 'tool_result',
+            tool_use_id: payload.call_id || null,
+            preview: previewText(typeof payload.output === 'string' ? payload.output : JSON.stringify(payload.output ?? ''), 240),
+            at: ts,
+        });
+        return;
+    }
+    if (payload.type === 'reasoning') {
+        // Reasoning items are first-class in Codex — we keep a stub on the
+        // timeline so downstream tools can correlate cost with thinking.
+        s.messages.push({ role: 'reasoning', preview: '(reasoning)', at: ts });
+        return;
+    }
+    if (role && Array.isArray(payload.content)) {
+        s.messages.push({
+            role,
+            preview: extractTextPreview(payload.content),
+            at: ts,
+        });
+    }
+}
+function extractTextPreview(content) {
+    for (const c of content) {
+        if (!c)
+            continue;
+        if (typeof c === 'string')
+            return previewText(c);
+        const part = c;
+        if (part.type === 'input_text' && typeof part.text === 'string')
+            return previewText(part.text);
+        if (part.type === 'output_text' && typeof part.text === 'string')
+            return previewText(part.text);
+        if (part.type === 'text' && typeof part.text === 'string')
+            return previewText(part.text);
+    }
+    return '';
+}
+function safeToolTarget(p) {
+    // Codex tools include `local_shell` (command exec) and a generic `apply_patch`
+    // among many MCP tools. We try to extract a SHORT non-secret identifier.
+    try {
+        const args = typeof p.arguments === 'string'
+            ? JSON.parse(p.arguments)
+            : p.arguments;
+        if (!args || typeof args !== 'object')
+            return null;
+        if (p.name === 'local_shell' || p.name === 'shell') {
+            const cmd = typeof args.command === 'string'
+                ? args.command
+                : Array.isArray(args.command) ? args.command.join(' ') : '';
+            const head = cmd.trim().split(/\s+/)[0] || '';
+            return head ? head.slice(0, 60) : null;
+        }
+        if (typeof args.file_path === 'string')
+            return args.file_path.slice(0, 160);
+        if (typeof args.path === 'string')
+            return args.path.slice(0, 160);
+        if (typeof args.url === 'string') {
+            try {
+                return new URL(args.url).host;
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    catch { /* fall through */ }
+    return null;
+}
+function _handleEventMsg(state, payload, ts) {
+    const s = state.session;
+    s.eventMessages += 1;
+    if (!payload || typeof payload !== 'object')
+        return;
+    const type = payload.type;
+    if (type === 'token_count') {
+        // Payload shape: { type: 'token_count', info: { total_token_usage, last_token_usage, model_context_window } }
+        const last = payload.info?.last_token_usage;
+        if (last) {
+            const input = Number(last.input_tokens) || 0;
+            const output = Number(last.output_tokens) || 0;
+            const cachedInput = Number(last.cached_input_tokens) || 0;
+            const reasoning = Number(last.reasoning_output_tokens) || 0;
+            const total = Number(last.total_tokens) || (input + output + cachedInput + reasoning);
+            s.totals.input_tokens += input;
+            s.totals.output_tokens += output;
+            s.totals.cache_read_tokens += cachedInput;
+            s.totals.reasoning_tokens += reasoning;
+            // Turn snapshot — Codex emits one token_count per model turn. We use
+            // the event ordinal as a synthetic turn id since the payload doesn't
+            // carry one. The token-count event always corresponds to one turn.
+            const turnId = `t${s.turns.length + 1}`;
+            s.turns.push({
+                turnId,
+                inputTokens: input,
+                outputTokens: output,
+                cachedInputTokens: cachedInput,
+                reasoningTokens: reasoning,
+                total,
+                at: ts,
+            });
+            s.turnCount = s.turns.length;
+        }
+        return;
+    }
+    if (type === 'session_configured') {
+        if (!s.modelPrimary && payload.model)
+            s.modelPrimary = payload.model;
+        return;
+    }
+    if (type === 'agent_message') {
+        if (typeof payload.message === 'string') {
+            s.messages.push({ role: 'assistant', preview: previewText(payload.message), at: ts });
+        }
+        return;
+    }
+    if (type === 'user_message') {
+        if (typeof payload.message === 'string') {
+            s.messages.push({ role: 'user', preview: previewText(payload.message), at: ts });
+        }
+        return;
+    }
+}
+function _handleCompacted(state, payload, ts) {
+    const s = state.session;
+    s.compactedItems += 1;
+    s.messages.push({
+        role: 'system',
+        preview: '(compacted) ' + previewText(payload?.message || '', 180),
+        at: ts,
+    });
+}
+function _processLine(state, line, lineNo) {
+    if (!line || !line.trim())
+        return;
+    const rec = safeParse(line);
+    const s = state.session;
+    if (!rec || typeof rec !== 'object') {
+        s.skippedLines += 1;
+        return;
+    }
+    s.jsonlRecords += 1;
+    const ts = typeof rec.timestamp === 'string' ? rec.timestamp : null;
+    if (ts) {
+        if (!s.startedAt || ts < s.startedAt)
+            s.startedAt = ts;
+        if (!s.endedAt || ts > s.endedAt)
+            s.endedAt = ts;
+    }
+    // RolloutLine flattens `type` to the top level (serde flatten).
+    const type = rec.type;
+    const payload = rec.payload;
+    switch (type) {
+        case 'session_meta':
+            _handleSessionMeta(state, payload, ts);
+            break;
+        case 'response_item':
+            _handleResponseItem(state, payload, ts);
+            break;
+        case 'event_msg':
+            _handleEventMsg(state, payload, ts);
+            break;
+        case 'compacted':
+            _handleCompacted(state, payload, ts);
+            break;
+        case 'turn_context':
+            // No-op for now; turn_context carries the per-turn model + tool list
+            // but we don't yet use it for normalization.
+            break;
+        default:
+            // Unknown line type — counted in jsonlRecords but no further action.
+            break;
+    }
+}
+/**
+ * Parse an array of raw JSONL lines (in-memory). Returns the normalized
+ * session object. The server uses this so it never touches the filesystem.
+ */
+export function parseCodexSessionLines(lines, { sourceFile = null, sourceMtime = null } = {}) {
+    if (!Array.isArray(lines)) {
+        throw new TypeError('parseCodexSessionLines: lines must be an array of strings');
+    }
+    const state = _initState({ sourceFile, sourceMtime });
+    for (let i = 0; i < lines.length; i++) {
+        _processLine(state, lines[i], i + 1);
+    }
+    return state.session;
+}
+/**
+ * Stream-parse a Codex JSONL file from disk. Used by the CLI ingest path.
+ */
+export async function parseCodexSessionFile(filePath, { mtime } = {}) {
+    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    let actualMtime = mtime;
+    if (!actualMtime) {
+        try {
+            actualMtime = fs.statSync(filePath).mtime.toISOString();
+        }
+        catch { /* keep null */ }
+    }
+    const state = _initState({ sourceFile: filePath, sourceMtime: actualMtime });
+    let lineNo = 0;
+    for await (const line of rl) {
+        _processLine(state, line, ++lineNo);
+    }
+    return state.session;
+}

package/lib/code/ingest-codex.js

@@ -0,0 +1,244 @@
+// cli/lib/code/ingest-codex.js
+//
+// Codex JSONL backfill. Walks `codexSessionsDir` (default ~/.codex/sessions),
+// parses each rollout file via app/lib/codex/parser.js, and either:
+//
+//   (a) writes the normalized session JSON to an --out directory (default
+//       ~/.dashclaw/codex-sessions/) for later server upload, or
+//   (b) POSTs it to --endpoint <url> directly.
+//
+// Phase 3 ships option (a) — local-only. Option (b) becomes useful once
+// the server-side ingest route accepts source_host='codex-jsonl' (deferred
+// until the AgentLens absorption lands, to avoid editing files already
+// being modified by another agent).
+//
+// Output JSON shape is the raw parseCodexSessionFile return value. The
+// server, when it grows codex ingestion, will deserialize it and persist.
+//
+// Logs per file: { file, status, reason?, session_uuid?, turns?, tokens? }.
+// NEVER logs raw message text — could leak user transcripts.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { homedir } from 'node:os';
+
+// Vendored: the CLI is published without app/, so it can't import the app's
+// TypeScript parser directly. See codex-parser.vendored.js.
+import { parseCodexSessionFile } from './codex-parser.vendored.js';
+
+const MAX_FILE_BYTES = 100 * 1024 * 1024; // codex rollouts can grow large
+
+export function defaultCodexSessionsDir(env = process.env) {
+  if (env.CODEX_SESSIONS_DIR) return env.CODEX_SESSIONS_DIR;
+  const root = process.platform === 'win32'
+    ? (env.USERPROFILE || homedir())
+    : (env.HOME || homedir());
+  // Codex defaults to <codex_home>/sessions; codex_home defaults to ~/.codex.
+  return path.join(env.CODEX_HOME || path.join(root, '.codex'), 'sessions');
+}
+
+export function defaultCodexOutDir(env = process.env) {
+  if (env.DASHCLAW_CODEX_OUT_DIR) return env.DASHCLAW_CODEX_OUT_DIR;
+  const root = process.platform === 'win32'
+    ? (env.USERPROFILE || homedir())
+    : (env.HOME || homedir());
+  return path.join(root, '.dashclaw', 'codex-sessions');
+}
+
+function listJsonlFiles(rootDir) {
+  const out = [];
+  let entries;
+  try { entries = fs.readdirSync(rootDir, { withFileTypes: true }); }
+  catch { return out; }
+  for (const e of entries) {
+    const p = path.join(rootDir, e.name);
+    if (e.isDirectory()) {
+      // Recurse one level — codex doesn't currently nest, but archived
+      // sessions sometimes land in `<sessions>/archived/`.
+      let inner;
+      try { inner = fs.readdirSync(p, { withFileTypes: true }); }
+      catch { continue; }
+      for (const f of inner) {
+        if (f.isFile() && f.name.endsWith('.jsonl')) {
+          out.push(path.join(p, f.name));
+        }
+      }
+    } else if (e.isFile() && e.name.endsWith('.jsonl')) {
+      out.push(p);
+    }
+  }
+  return out;
+}
+
+function bytesOf(filePath) {
+  try { return fs.statSync(filePath).size; }
+  catch { return 0; }
+}
+
+/**
+ * Build the per-session output JSON. Stable shape that the server can
+ * consume once codex ingestion lands.
+ */
+function buildOutputPayload(session) {
+  return {
+    parser_version: session.parserVersion,
+    agent_kind: session.agentKind,
+    session_uuid: session.sessionUuid,
+    cwd: session.cwd,
+    cli_version: session.cliVersion,
+    originator: session.originator,
+    model_provider: session.modelProvider,
+    model_primary: session.modelPrimary,
+    started_at: session.startedAt,
+    ended_at: session.endedAt,
+    git_branch: session.gitBranch,
+    git_commit: session.gitCommit,
+    git_repo_url: session.gitRepoUrl,
+    jsonl_records: session.jsonlRecords,
+    skipped_lines: session.skippedLines,
+    response_items: session.responseItems,
+    event_messages: session.eventMessages,
+    compacted_items: session.compactedItems,
+    tool_call_count: session.toolCallCount,
+    turn_count: session.turnCount,
+    totals: session.totals,
+    turns: session.turns,
+    // Messages and tool uses are large — keep them in the payload so the
+    // server can persist them when ready. CLI does not log them.
+    messages: session.messages,
+    tool_uses: session.toolUses,
+    source: {
+      host: 'codex-jsonl',
+      file: session.sourceFile,
+      mtime: session.sourceMtime,
+    },
+  };
+}
+
+async function writeLocalOut({ outDir, payload }) {
+  fs.mkdirSync(outDir, { recursive: true });
+  const id = payload.session_uuid || `unknown-${Date.now()}`;
+  const target = path.join(outDir, `${id}.json`);
+  fs.writeFileSync(target, JSON.stringify(payload, null, 2) + '\n');
+  return target;
+}
+
+async function postPayload({ endpoint, apiKey, payload, timeoutMs = 10000 }) {
+  const url = new URL(endpoint);
+  const body = JSON.stringify(payload);
+  const headers = {
+    'content-type': 'application/json',
+    'x-api-key': apiKey,
+  };
+
+  const { request: httpReq } = await import('node:http');
+  const { request: httpsReq } = await import('node:https');
+  const lib = url.protocol === 'https:' ? httpsReq : httpReq;
+
+  return new Promise((resolve) => {
+    const req = lib({
+      method: 'POST',
+      host: url.hostname,
+      port: url.port || (url.protocol === 'https:' ? 443 : 80),
+      path: url.pathname + url.search,
+      headers: { ...headers, 'content-length': Buffer.byteLength(body).toString() },
+    }, (res) => {
+      let chunks = '';
+      res.setEncoding('utf8');
+      res.on('data', (c) => { chunks += c; });
+      res.on('end', () => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          resolve({ status: 'ingested', http: res.statusCode });
+        } else {
+          resolve({ status: 'error', reason: `http_${res.statusCode}`, detail: chunks.slice(0, 200) });
+        }
+      });
+    });
+    req.on('error', (err) => resolve({ status: 'error', reason: 'network', detail: err.message }));
+    req.setTimeout(timeoutMs, () => { req.destroy(); resolve({ status: 'error', reason: 'timeout' }); });
+    req.write(body);
+    req.end();
+  });
+}
+
+/**
+ * Top-level orchestration. Returns an array of result objects, one per
+ * jsonl file found in `sessionsDir`.
+ */
+export async function runCodexIngest({
+  sessionsDir = defaultCodexSessionsDir(),
+  outDir = defaultCodexOutDir(),
+  endpoint = null,    // when set, POST instead of write-local
+  apiKey = null,
+  dryRun = false,
+  logger = console,
+} = {}) {
+  if (!fs.existsSync(sessionsDir)) {
+    logger.warn?.(`codex ingest: sessions dir does not exist: ${sessionsDir}`);
+    return [];
+  }
+  const files = listJsonlFiles(sessionsDir);
+  if (files.length === 0) {
+    logger.info?.(`codex ingest: no .jsonl files in ${sessionsDir}`);
+    return [];
+  }
+
+  const results = [];
+  for (const file of files) {
+    const size = bytesOf(file);
+    if (size === 0) {
+      results.push({ file, status: 'skipped', reason: 'empty' });
+      continue;
+    }
+    if (size > MAX_FILE_BYTES) {
+      results.push({ file, status: 'skipped', reason: 'too_large', bytes: size });
+      continue;
+    }
+
+    let session;
+    try {
+      session = await parseCodexSessionFile(file);
+    } catch (err) {
+      results.push({ file, status: 'error', reason: 'parse', detail: err.message });
+      continue;
+    }
+    const payload = buildOutputPayload(session);
+
+    if (dryRun) {
+      results.push({
+        file,
+        status: 'dry_run',
+        session_uuid: session.sessionUuid,
+        turns: session.turnCount,
+        tokens: session.totals,
+      });
+      continue;
+    }
+
+    if (endpoint) {
+      const httpResult = await postPayload({ endpoint, apiKey, payload });
+      results.push({
+        file,
+        ...httpResult,
+        session_uuid: session.sessionUuid,
+        turns: session.turnCount,
+      });
+    } else {
+      try {
+        const target = await writeLocalOut({ outDir, payload });
+        results.push({
+          file,
+          status: 'written_local',
+          session_uuid: session.sessionUuid,
+          out: target,
+          turns: session.turnCount,
+          tokens: session.totals,
+        });
+      } catch (err) {
+        results.push({ file, status: 'error', reason: 'write', detail: err.message });
+      }
+    }
+  }
+
+  return results;
+}