@ekkos/cli 0.2.7 → 0.2.8

package/dist/capture/stream-tailer.js

@@ -0,0 +1,658 @@
+"use strict";
+/**
+ * JSONL Stream Tailer for Claude Code transcripts
+ *
+ * Tails ~/.claude/projects/{encoded-path}/{session-uuid}.jsonl
+ * and extracts assistant_stream + tool events in real-time.
+ *
+ * This is the core component for "near-zero context loss" on /continue.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamTailer = void 0;
+exports.reconstructTurnFromEvents = reconstructTurnFromEvents;
+exports.middleTruncate = middleTruncate;
+exports.detectOpenLoops = detectOpenLoops;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const events_1 = require("events");
+const crypto = __importStar(require("crypto"));
+const POLL_INTERVAL_ACTIVE = 100; // ms while actively receiving
+const POLL_INTERVAL_IDLE = 500; // ms when idle
+const IDLE_THRESHOLD = 5000; // ms without activity = idle
+const CHECKPOINT_INTERVAL = 5000; // Save checkpoint every 5 seconds
+const HEAD_SIZE = 512; // First N chars to keep
+const TAIL_SIZE = 8192; // Last N chars to keep (8KB)
+class StreamTailer extends events_1.EventEmitter {
+    /**
+     * Generate a stable event_id for idempotency
+     *
+     * CRITICAL: event_id must be stable for the same underlying transcript data.
+     * Using transcript_line_uuid + block_index ensures:
+     * - Same ID after tailer restart
+     * - De-dupe when reading overlapping transcript ranges
+     * - True idempotency for append operations
+     *
+     * Format: {kind_prefix}_{transcript_line_uuid}_{block_index}_{content_hash}
+     */
+    generateEventId(kind, turnId, transcriptLineUuid, blockIndex, contentHash) {
+        // Fallback to counter if no UUID (legacy compatibility)
+        if (!transcriptLineUuid) {
+            const counter = ++this.eventCounter;
+            const hash = crypto.createHash('sha256')
+                .update(`${this.state.sessionId}:${kind}:${turnId}:${counter}`)
+                .digest('hex')
+                .slice(0, 12);
+            return `${kind.slice(0, 3)}_${counter}_${hash}`;
+        }
+        // Stable ID from transcript identity
+        const uuidShort = transcriptLineUuid.slice(0, 8);
+        const blockIdx = blockIndex ?? 0;
+        const hashPart = contentHash
+            ? contentHash.slice(0, 8)
+            : crypto.createHash('sha256')
+                .update(`${transcriptLineUuid}:${kind}:${blockIdx}`)
+                .digest('hex')
+                .slice(0, 8);
+        return `${kind.slice(0, 3)}_${uuidShort}_${blockIdx}_${hashPart}`;
+    }
+    /**
+     * Generate content hash for delta events
+     */
+    hashContent(content) {
+        return crypto.createHash('sha256')
+            .update(content)
+            .digest('hex')
+            .slice(0, 12);
+    }
+    /**
+     * Initialize state machine with default values
+     */
+    initStateMachine() {
+        return {
+            session_id: this.state.sessionId,
+            session_name: this.state.sessionName,
+            state: 'idle',
+            state_entered_at: new Date().toISOString(),
+            last_complete_turn_id: 0,
+            current_turn_id: 0,
+            in_progress_text_head: '',
+            in_progress_text_tail: '',
+            in_progress_total_chars: 0,
+            open_loops: [],
+            last_event_ts: new Date().toISOString(),
+            last_checkpoint_ts: new Date().toISOString(),
+            stream_bytes_captured: 0,
+            events_captured: 0,
+        };
+    }
+    /**
+     * Update state machine state
+     */
+    updateState(newState) {
+        if (this.stateMachine.state !== newState) {
+            this.stateMachine.state = newState;
+            this.stateMachine.state_entered_at = new Date().toISOString();
+        }
+    }
+    /**
+     * Update in-progress text head/tail
+     */
+    updateInProgressText(delta) {
+        this.inProgressText += delta;
+        this.stateMachine.in_progress_total_chars = this.inProgressText.length;
+        this.stateMachine.in_progress_text_head = this.inProgressText.slice(0, HEAD_SIZE);
+        this.stateMachine.in_progress_text_tail = this.inProgressText.slice(-TAIL_SIZE);
+    }
+    /**
+     * Clear in-progress text (on turn seal)
+     */
+    clearInProgressText() {
+        this.inProgressText = '';
+        this.stateMachine.in_progress_text_head = '';
+        this.stateMachine.in_progress_text_tail = '';
+        this.stateMachine.in_progress_total_chars = 0;
+    }
+    /**
+     * Add open loop
+     */
+    addOpenLoop(type, id, name) {
+        this.stateMachine.open_loops.push({
+            type,
+            id,
+            name,
+            started_at: new Date().toISOString(),
+        });
+    }
+    /**
+     * Close open loop
+     */
+    closeOpenLoop(id) {
+        this.stateMachine.open_loops = this.stateMachine.open_loops.filter(l => l.id !== id);
+    }
+    /**
+     * Save state machine to disk
+     */
+    saveStateMachine() {
+        try {
+            this.stateMachine.last_event_ts = new Date().toISOString();
+            fs.writeFileSync(this.stateMachinePath, JSON.stringify(this.stateMachine, null, 2));
+        }
+        catch (err) {
+            this.emit('persist_error', err);
+        }
+    }
+    /**
+     * Save checkpoint
+     */
+    saveCheckpoint() {
+        if (this.stateMachine.state !== 'assistant_streaming' && this.stateMachine.state !== 'tool_running') {
+            // Only checkpoint during active work
+            return;
+        }
+        try {
+            const checkpoint = {
+                session_id: this.state.sessionId,
+                turn_id: this.state.currentTurnId,
+                checkpoint_ts: new Date().toISOString(),
+                text_head: this.stateMachine.in_progress_text_head,
+                text_tail: this.stateMachine.in_progress_text_tail,
+                state: this.stateMachine.state,
+                open_loops: this.stateMachine.open_loops.map(l => ({
+                    type: l.type,
+                    id: l.id,
+                    name: l.name,
+                    started_at: l.started_at,
+                })),
+                total_chars: this.stateMachine.in_progress_total_chars,
+                events_since_last_checkpoint: this.eventsSinceCheckpoint,
+            };
+            fs.writeFileSync(this.checkpointPath, JSON.stringify(checkpoint, null, 2));
+            this.stateMachine.last_checkpoint_ts = checkpoint.checkpoint_ts;
+            this.eventsSinceCheckpoint = 0;
+        }
+        catch (err) {
+            this.emit('persist_error', err);
+        }
+    }
+    constructor(options) {
+        super();
+        this.pollTimer = null;
+        this.checkpointTimer = null;
+        this.running = false;
+        this.eventCounter = 0; // Monotonic counter for event_id generation
+        this.inProgressText = ''; // Full in-progress text for head/tail extraction
+        this.eventsSinceCheckpoint = 0;
+        // Idempotency tracking - set of seen event_ids
+        this.seenEventIds = new Set();
+        this.state = {
+            transcriptPath: options.transcriptPath,
+            sessionId: options.sessionId,
+            sessionName: options.sessionName,
+            readOffset: 0,
+            pendingBuffer: '',
+            currentTurnId: 0,
+            currentTurnStatus: 'complete',
+            lastActivityTs: Date.now(),
+        };
+        this.cacheDir = options.cacheDir;
+        this.eventLogPath = path.join(this.cacheDir, `${options.sessionId}.stream.jsonl`);
+        this.stateMachinePath = path.join(this.cacheDir, `${options.sessionId}.state.json`);
+        this.checkpointPath = path.join(this.cacheDir, `${options.sessionId}.checkpoint.json`);
+        this.onEventCallback = options.onEvent;
+        this.stateMachine = this.initStateMachine();
+    }
+    /**
+     * Start tailing the transcript file
+     */
+    async start() {
+        if (this.running)
+            return;
+        this.running = true;
+        // Ensure cache directory exists
+        await fs.promises.mkdir(this.cacheDir, { recursive: true });
+        // Emit session start event
+        this.emitEvent({
+            kind: 'session_start',
+            event_id: this.generateEventId('session_start', 0),
+            session_id: this.state.sessionId,
+            session_name: this.state.sessionName,
+            ts: new Date().toISOString(),
+        });
+        // Save initial state machine
+        this.saveStateMachine();
+        // Start checkpoint timer
+        this.checkpointTimer = setInterval(() => {
+            this.saveCheckpoint();
+            this.saveStateMachine();
+        }, CHECKPOINT_INTERVAL);
+        // Start polling
+        this.schedulePoll();
+        this.emit('started', { sessionId: this.state.sessionId });
+    }
+    /**
+     * Stop tailing
+     */
+    stop() {
+        this.running = false;
+        if (this.pollTimer) {
+            clearTimeout(this.pollTimer);
+            this.pollTimer = null;
+        }
+        if (this.checkpointTimer) {
+            clearInterval(this.checkpointTimer);
+            this.checkpointTimer = null;
+        }
+        // Save final state
+        this.updateState('idle');
+        this.saveStateMachine();
+        this.saveCheckpoint();
+        this.emit('stopped', { sessionId: this.state.sessionId });
+    }
+    /**
+     * Get current state (for debugging/status)
+     */
+    getState() {
+        return { ...this.state };
+    }
+    schedulePoll() {
+        if (!this.running)
+            return;
+        const timeSinceActivity = Date.now() - this.state.lastActivityTs;
+        const interval = timeSinceActivity > IDLE_THRESHOLD
+            ? POLL_INTERVAL_IDLE
+            : POLL_INTERVAL_ACTIVE;
+        this.pollTimer = setTimeout(() => this.poll(), interval);
+    }
+    async poll() {
+        if (!this.running)
+            return;
+        try {
+            await this.readNewContent();
+        }
+        catch (err) {
+            // File might not exist yet or be temporarily unavailable
+            if (err.code !== 'ENOENT') {
+                this.emit('error', err);
+            }
+        }
+        this.schedulePoll();
+    }
+    async readNewContent() {
+        const stat = await fs.promises.stat(this.state.transcriptPath);
+        const fileSize = stat.size;
+        // Handle file truncation (rotation/reset)
+        if (fileSize < this.state.readOffset) {
+            this.state.readOffset = 0;
+            this.state.pendingBuffer = '';
+            this.emit('truncated', { sessionId: this.state.sessionId });
+        }
+        // No new content
+        if (fileSize <= this.state.readOffset) {
+            return;
+        }
+        // Read new bytes
+        const fd = await fs.promises.open(this.state.transcriptPath, 'r');
+        try {
+            const bytesToRead = fileSize - this.state.readOffset;
+            const buffer = Buffer.alloc(bytesToRead);
+            await fd.read(buffer, 0, bytesToRead, this.state.readOffset);
+            this.state.readOffset = fileSize;
+            // Append to pending buffer and process complete lines
+            this.state.pendingBuffer += buffer.toString('utf-8');
+            this.processBuffer();
+            this.state.lastActivityTs = Date.now();
+        }
+        finally {
+            await fd.close();
+        }
+    }
+    processBuffer() {
+        // Split on newlines, keep incomplete line in buffer
+        const lines = this.state.pendingBuffer.split('\n');
+        this.state.pendingBuffer = lines.pop() || ''; // Last element is incomplete or empty
+        for (const line of lines) {
+            if (!line.trim())
+                continue;
+            this.processLine(line);
+        }
+    }
+    processLine(line) {
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            // Malformed line, skip
+            this.emit('parse_error', { line: line.slice(0, 100) });
+            return;
+        }
+        // Skip non-message types (file-history-snapshot, etc.)
+        if (!parsed.type || !['user', 'assistant'].includes(parsed.type)) {
+            return;
+        }
+        const ts = parsed.timestamp || new Date().toISOString();
+        if (parsed.type === 'user') {
+            this.handleUserEvent(parsed, ts);
+        }
+        else if (parsed.type === 'assistant') {
+            this.handleAssistantEvent(parsed, ts);
+        }
+    }
+    handleUserEvent(line, ts) {
+        // Seal any in_progress assistant turn
+        if (this.state.currentTurnStatus === 'in_progress') {
+            // Use the incoming user message UUID as the seal trigger (stable)
+            this.emitEvent({
+                kind: 'seal_turn',
+                event_id: this.generateEventId('seal_turn', this.state.currentTurnId, line.uuid, // Use user message UUID as seal trigger
+                0, 'user_boundary'),
+                turn_id: this.state.currentTurnId,
+                reason: 'user_boundary',
+                ts,
+            });
+            this.state.currentTurnStatus = 'complete';
+            // Update state machine
+            this.stateMachine.last_complete_turn_id = this.state.currentTurnId;
+            this.clearInProgressText();
+            this.stateMachine.open_loops = []; // Clear open loops on turn seal
+        }
+        // Start new turn
+        this.state.currentTurnId++;
+        // Update state machine
+        this.stateMachine.current_turn_id = this.state.currentTurnId;
+        this.updateState('idle');
+        // Extract user query
+        let query = '';
+        if (typeof line.message.content === 'string') {
+            query = line.message.content;
+        }
+        else if (Array.isArray(line.message.content)) {
+            // Extract text blocks
+            query = line.message.content
+                .filter((b) => b.type === 'text')
+                .map((b) => b.text || '')
+                .join('\n');
+        }
+        // Skip meta messages (command caveats, etc.) unless they have real content
+        if (line.isMeta && !query.replace(/<[^>]*>/g, '').trim()) {
+            return;
+        }
+        // Use transcript_line_uuid for stable event_id
+        this.emitEvent({
+            kind: 'user_query',
+            event_id: this.generateEventId('user_query', this.state.currentTurnId, line.uuid, 0, this.hashContent(query)),
+            turn_id: this.state.currentTurnId,
+            query,
+            ts,
+            transcript_line_uuid: line.uuid,
+        });
+        this.emit('user_turn', { turnId: this.state.currentTurnId, query });
+    }
+    handleAssistantEvent(line, ts) {
+        // Mark turn as in_progress
+        this.state.currentTurnStatus = 'in_progress';
+        const content = line.message.content;
+        if (!Array.isArray(content))
+            return;
+        let textDelta = '';
+        let deltaOffset = 0;
+        let blockIndex = 0; // Track block index within this transcript line for stable event_id
+        for (const block of content) {
+            switch (block.type) {
+                case 'text':
+                    if (block.text) {
+                        textDelta += block.text;
+                        // Update state machine
+                        this.updateState('assistant_streaming');
+                        this.updateInProgressText(block.text);
+                        this.eventsSinceCheckpoint++;
+                        // Generate stable event_id using transcript_line_uuid + block_index + content_hash
+                        const contentHash = this.hashContent(block.text);
+                        this.emitEvent({
+                            kind: 'assistant_text_delta',
+                            event_id: this.generateEventId('assistant_text_delta', this.state.currentTurnId, line.uuid, blockIndex, contentHash),
+                            turn_id: this.state.currentTurnId,
+                            delta: block.text,
+                            offset: deltaOffset,
+                            ts,
+                            transcript_line_uuid: line.uuid,
+                        });
+                        deltaOffset += block.text.length;
+                    }
+                    break;
+                case 'tool_use':
+                    if (block.id && block.name) {
+                        // Update state machine - add open loop
+                        this.updateState('tool_running');
+                        this.addOpenLoop('tool', block.id, block.name);
+                        this.eventsSinceCheckpoint++;
+                        // tool_use.id is stable, use it for idempotency
+                        this.emitEvent({
+                            kind: 'tool_use',
+                            event_id: this.generateEventId('tool_use', this.state.currentTurnId, line.uuid, blockIndex, block.id // tool_use.id is already stable
+                            ),
+                            turn_id: this.state.currentTurnId,
+                            id: block.id,
+                            name: block.name,
+                            input: block.input,
+                            ts,
+                            transcript_line_uuid: line.uuid,
+                        });
+                    }
+                    break;
+                case 'tool_result':
+                    if (block.tool_use_id) {
+                        // Update state machine - close open loop
+                        this.closeOpenLoop(block.tool_use_id);
+                        if (this.stateMachine.open_loops.length === 0) {
+                            this.updateState('assistant_streaming');
+                        }
+                        this.eventsSinceCheckpoint++;
+                        // tool_use_id is stable, use it for idempotency
+                        this.emitEvent({
+                            kind: 'tool_result',
+                            event_id: this.generateEventId('tool_result', this.state.currentTurnId, line.uuid, blockIndex, block.tool_use_id // tool_use_id is already stable
+                            ),
+                            turn_id: this.state.currentTurnId,
+                            id: block.tool_use_id,
+                            output: block.content,
+                            is_error: block.is_error,
+                            ts,
+                            transcript_line_uuid: line.uuid,
+                        });
+                    }
+                    break;
+                case 'thinking':
+                    // Ignore thinking blocks (encrypted, not needed for restore)
+                    break;
+            }
+            blockIndex++; // Increment for each block processed
+        }
+        // Update metrics
+        this.stateMachine.stream_bytes_captured += textDelta.length;
+        this.stateMachine.events_captured++;
+        if (textDelta) {
+            this.emit('assistant_delta', {
+                turnId: this.state.currentTurnId,
+                delta: textDelta,
+            });
+        }
+    }
+    emitEvent(event) {
+        // Idempotency check - skip if we've already seen this event_id
+        if (this.seenEventIds.has(event.event_id)) {
+            return; // De-dupe: event already processed
+        }
+        this.seenEventIds.add(event.event_id);
+        // Prune old event_ids to prevent unbounded memory growth
+        // Keep last 10000 event_ids (more than enough for a session)
+        if (this.seenEventIds.size > 10000) {
+            const idsArray = Array.from(this.seenEventIds);
+            this.seenEventIds = new Set(idsArray.slice(-5000));
+        }
+        // Append to local event log (sync for reliability)
+        try {
+            fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n', 'utf-8');
+        }
+        catch (err) {
+            this.emit('persist_error', err);
+        }
+        // Callback if provided
+        if (this.onEventCallback) {
+            this.onEventCallback(event);
+        }
+        // Emit on EventEmitter
+        this.emit('event', event);
+    }
+}
+exports.StreamTailer = StreamTailer;
+/**
+ * Utility: Reconstruct StreamTurn from event log
+ */
+async function reconstructTurnFromEvents(eventLogPath, turnId) {
+    const turns = new Map();
+    let latestTurnId = 0;
+    try {
+        const content = await fs.promises.readFile(eventLogPath, 'utf-8');
+        const lines = content.split('\n').filter(l => l.trim());
+        for (const line of lines) {
+            try {
+                const event = JSON.parse(line);
+                if (event.kind === 'user_query') {
+                    if (!turns.has(event.turn_id)) {
+                        turns.set(event.turn_id, {
+                            user_query: '',
+                            assistant_stream: '',
+                            tools: [],
+                            status: 'in_progress',
+                        });
+                    }
+                    turns.get(event.turn_id).user_query = event.query;
+                    latestTurnId = Math.max(latestTurnId, event.turn_id);
+                }
+                if (event.kind === 'assistant_text_delta') {
+                    if (!turns.has(event.turn_id)) {
+                        turns.set(event.turn_id, {
+                            user_query: '',
+                            assistant_stream: '',
+                            tools: [],
+                            status: 'in_progress',
+                        });
+                    }
+                    turns.get(event.turn_id).assistant_stream += event.delta;
+                    latestTurnId = Math.max(latestTurnId, event.turn_id);
+                }
+                if (event.kind === 'tool_use') {
+                    if (turns.has(event.turn_id)) {
+                        turns.get(event.turn_id).tools.push({
+                            kind: 'tool_use',
+                            id: event.id,
+                            name: event.name,
+                            ts: event.ts,
+                        });
+                    }
+                }
+                if (event.kind === 'tool_result') {
+                    if (turns.has(event.turn_id)) {
+                        turns.get(event.turn_id).tools.push({
+                            kind: 'tool_result',
+                            id: event.id,
+                            ts: event.ts,
+                        });
+                    }
+                }
+                if (event.kind === 'seal_turn') {
+                    if (turns.has(event.turn_id)) {
+                        turns.get(event.turn_id).status = 'complete';
+                    }
+                }
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+    }
+    catch {
+        // File doesn't exist or read error
+    }
+    // Filter to specific turn if requested
+    if (turnId !== undefined) {
+        for (const [id] of turns) {
+            if (id !== turnId)
+                turns.delete(id);
+        }
+    }
+    return { turns, latestTurnId };
+}
+/**
+ * Utility: Middle-truncate text (keep head + tail)
+ */
+function middleTruncate(text, opts = {}) {
+    const { headChars = 2000, tailChars = 8000, maxTotal = 12000 } = opts;
+    if (text.length <= maxTotal)
+        return text;
+    const head = text.slice(0, headChars);
+    const tail = text.slice(-tailChars);
+    const truncatedCount = text.length - headChars - tailChars;
+    return `${head}\n\n[...${truncatedCount} chars truncated...]\n\n${tail}`;
+}
+/**
+ * Utility: Detect open loops (incomplete tools)
+ */
+function detectOpenLoops(tools) {
+    const openLoops = [];
+    const toolUseIds = new Set();
+    const toolResultIds = new Set();
+    for (const tool of tools) {
+        if (tool.kind === 'tool_use') {
+            toolUseIds.add(tool.id);
+        }
+        if (tool.kind === 'tool_result') {
+            toolResultIds.add(tool.id);
+        }
+    }
+    // Find tool_use without corresponding tool_result
+    for (const id of toolUseIds) {
+        if (!toolResultIds.has(id)) {
+            const tool = tools.find(t => t.kind === 'tool_use' && t.id === id);
+            openLoops.push({
+                type: 'incomplete_tool',
+                detail: `Tool ${tool?.name || 'unknown'} (${id}) started but did not complete`,
+            });
+        }
+    }
+    return openLoops;
+}