@ekkos/cli 0.2.18 → 1.0.0

package/dist/capture/eviction-client.js

@@ -0,0 +1,454 @@
+"use strict";
+/**
+ * EVICTION CLIENT
+ * ================
+ *
+ * Client module for the Handshake Eviction Protocol.
+ * Provides functions to call ekkOS_EvictPrepare and ekkOS_EvictConfirm
+ * ensuring zero data loss during context eviction.
+ *
+ * Protocol:
+ * 1. prepareEviction() - Write to R2, get ACK
+ * 2. (caller deletes locally)
+ * 3. confirmEviction() - Mark as committed
+ *
+ * Features:
+ * - Retry with exponential backoff
+ * - Timeout handling
+ * - Fallback detection
+ * - Idempotency via clientNonce
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queueEvictionForRetry = queueEvictionForRetry;
+exports.drainRetryQueue = drainRetryQueue;
+exports.getRetryQueueStats = getRetryQueueStats;
+exports.createEvictionId = createEvictionId;
+exports.createFingerprint = createFingerprint;
+exports.checkEvictionHealth = checkEvictionHealth;
+exports.prepareEviction = prepareEviction;
+exports.confirmEviction = confirmEviction;
+exports.handshakeEviction = handshakeEviction;
+const crypto_1 = require("crypto");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+// ═══════════════════════════════════════════════════════════════════════════
+// RETRY QUEUE - Auto-sync when R2 reconnects
+// ═══════════════════════════════════════════════════════════════════════════
+//
+// When handshake fails (R2/proxy down), evictions are queued locally.
+// On next successful handshake, the queue is drained automatically.
+// Bounded: max 50 entries, oldest dropped on overflow.
+//
+const RETRY_QUEUE_PATH = path.join(os.homedir(), '.ekkos', 'eviction-retry-queue.jsonl');
+const MAX_RETRY_QUEUE_SIZE = 50;
+let isDraining = false;
+/**
+ * Queue a failed eviction for retry when R2 reconnects
+ */
+function queueEvictionForRetry(messages, indices, estimatedTokens, reason, options) {
+    try {
+        const dir = path.dirname(RETRY_QUEUE_PATH);
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true });
+        const entry = {
+            messages,
+            indices,
+            estimatedTokens,
+            reason,
+            options,
+            queuedAt: Date.now(),
+            attempts: 0,
+        };
+        // Read existing queue
+        let entries = [];
+        if (fs.existsSync(RETRY_QUEUE_PATH)) {
+            entries = fs.readFileSync(RETRY_QUEUE_PATH, 'utf-8')
+                .split('\n')
+                .filter(l => l.trim());
+        }
+        // Enforce max size (drop oldest)
+        if (entries.length >= MAX_RETRY_QUEUE_SIZE) {
+            entries = entries.slice(entries.length - MAX_RETRY_QUEUE_SIZE + 1);
+        }
+        entries.push(JSON.stringify(entry));
+        fs.writeFileSync(RETRY_QUEUE_PATH, entries.join('\n') + '\n');
+        console.log(`[EvictionRetry] Queued eviction for retry (${messages.length} msgs, queue size: ${entries.length})`);
+    }
+    catch (err) {
+        console.warn('[EvictionRetry] Failed to queue:', err instanceof Error ? err.message : err);
+    }
+}
+/**
+ * Drain the retry queue — called after a successful handshake
+ * Non-blocking: runs in background, doesn't delay current eviction
+ */
+async function drainRetryQueue(apiUrl, authToken) {
+    if (isDraining)
+        return { drained: 0, failed: 0, remaining: -1 };
+    if (!fs.existsSync(RETRY_QUEUE_PATH))
+        return { drained: 0, failed: 0, remaining: 0 };
+    isDraining = true;
+    let drained = 0;
+    let failed = 0;
+    try {
+        const content = fs.readFileSync(RETRY_QUEUE_PATH, 'utf-8');
+        const lines = content.split('\n').filter(l => l.trim());
+        if (lines.length === 0) {
+            isDraining = false;
+            return { drained: 0, failed: 0, remaining: 0 };
+        }
+        console.log(`[EvictionRetry] Draining ${lines.length} queued evictions...`);
+        const remaining = [];
+        for (const line of lines) {
+            try {
+                const entry = JSON.parse(line);
+                // Skip entries older than 24 hours (data likely stale)
+                if (Date.now() - entry.queuedAt > 24 * 60 * 60 * 1000) {
+                    console.log(`[EvictionRetry] Dropping stale entry (${((Date.now() - entry.queuedAt) / 3600000).toFixed(1)}h old)`);
+                    continue;
+                }
+                // Skip entries that failed too many times
+                if (entry.attempts >= 3) {
+                    console.log(`[EvictionRetry] Dropping entry after 3 failed attempts`);
+                    continue;
+                }
+                // Attempt handshake
+                const result = await handshakeEviction(entry.messages, entry.indices, entry.estimatedTokens, entry.reason, { ...entry.options, apiUrl, authToken });
+                if (result.success) {
+                    drained++;
+                    console.log(`[EvictionRetry] Successfully synced queued eviction ${result.evictionId}`);
+                }
+                else {
+                    failed++;
+                    entry.attempts++;
+                    remaining.push(JSON.stringify(entry));
+                }
+            }
+            catch {
+                failed++;
+                remaining.push(line); // Keep unparseable entries for manual review
+            }
+        }
+        // Rewrite queue with remaining entries
+        if (remaining.length > 0) {
+            fs.writeFileSync(RETRY_QUEUE_PATH, remaining.join('\n') + '\n');
+        }
+        else {
+            // Queue fully drained — remove file
+            try {
+                fs.unlinkSync(RETRY_QUEUE_PATH);
+            }
+            catch { /* ok */ }
+        }
+        console.log(`[EvictionRetry] Drain complete: ${drained} synced, ${failed} failed, ${remaining.length} remaining`);
+        return { drained, failed, remaining: remaining.length };
+    }
+    catch (err) {
+        console.warn('[EvictionRetry] Drain error:', err instanceof Error ? err.message : err);
+        return { drained, failed, remaining: -1 };
+    }
+    finally {
+        isDraining = false;
+    }
+}
+/**
+ * Get retry queue stats (for monitoring)
+ */
+function getRetryQueueStats() {
+    try {
+        if (!fs.existsSync(RETRY_QUEUE_PATH))
+            return { size: 0, oldestAge: null };
+        const content = fs.readFileSync(RETRY_QUEUE_PATH, 'utf-8');
+        const lines = content.split('\n').filter(l => l.trim());
+        if (lines.length === 0)
+            return { size: 0, oldestAge: null };
+        const oldest = JSON.parse(lines[0]);
+        return {
+            size: lines.length,
+            oldestAge: Date.now() - oldest.queuedAt,
+        };
+    }
+    catch {
+        return { size: 0, oldestAge: null };
+    }
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// CONFIGURATION
+// ═══════════════════════════════════════════════════════════════════════════
+const EVICTION_TIMEOUT_MS = 30000; // 30 second timeout
+const MAX_RETRIES = 3;
+const HEALTH_CHECK_TIMEOUT_MS = 5000;
+// ═══════════════════════════════════════════════════════════════════════════
+// UTILITY FUNCTIONS
+// ═══════════════════════════════════════════════════════════════════════════
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/**
+ * Create deterministic eviction ID from messages
+ * Same messages = same evictionId (enables dedup)
+ */
+function createEvictionId(messages) {
+    const data = JSON.stringify(messages);
+    return (0, crypto_1.createHash)('sha256').update(data).digest('hex').slice(0, 12);
+}
+/**
+ * Create fingerprint for a message
+ */
+function createFingerprint(msg) {
+    const data = msg.role + JSON.stringify(msg.content);
+    return (0, crypto_1.createHash)('sha256').update(data).digest('hex').slice(0, 16);
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// HEALTH CHECK
+// ═══════════════════════════════════════════════════════════════════════════
+/**
+ * Check if the eviction API is available
+ * Returns true if proxy is reachable and healthy
+ */
+async function checkEvictionHealth(apiUrl, authToken) {
+    try {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS);
+        const response = await fetch(`${apiUrl}/api/v1/mcp/health`, {
+            method: 'GET',
+            headers: {
+                'Authorization': `Bearer ${authToken}`,
+            },
+            signal: controller.signal,
+        });
+        clearTimeout(timeoutId);
+        return response.ok;
+    }
+    catch {
+        return false;
+    }
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// PREPARE EVICTION (PHASE 1)
+// ═══════════════════════════════════════════════════════════════════════════
+/**
+ * Prepare eviction by writing to R2 via the proxy.
+ * Returns ACK with evictionId if successful.
+ * Caller must NOT delete locally until this succeeds.
+ */
+async function prepareEviction(apiUrl, authToken, sessionId, sessionName, userId, tenantId, messages, manifest, projectPath) {
+    const clientNonce = (0, crypto_1.randomUUID)();
+    for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+        try {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), EVICTION_TIMEOUT_MS);
+            const response = await fetch(`${apiUrl}/api/v1/mcp/call`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${authToken}`,
+                },
+                body: JSON.stringify({
+                    tool: 'ekkOS_EvictPrepare',
+                    args: {
+                        sessionId,
+                        sessionName,
+                        userId,
+                        tenantId,
+                        projectPath: projectPath || process.cwd(),
+                        messages,
+                        manifest,
+                        clientNonce,
+                    },
+                }),
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                const errorText = await response.text().catch(() => 'Unknown error');
+                throw new Error(`HTTP ${response.status}: ${errorText}`);
+            }
+            const data = await response.json();
+            if (!data.success) {
+                throw new Error(data.error || 'Prepare failed');
+            }
+            const result = data.result || data;
+            return {
+                success: true,
+                evictionId: result.evictionId || manifest.evictionId,
+                r2Key: result.r2Key,
+                bytesWritten: result.bytesWritten,
+                checksum: result.checksum,
+                status: result.status,
+                clientNonce,
+            };
+        }
+        catch (err) {
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            console.warn(`[EvictionClient] Prepare attempt ${attempt}/${MAX_RETRIES} failed: ${errorMessage}`);
+            if (attempt === MAX_RETRIES) {
+                return {
+                    success: false,
+                    evictionId: manifest.evictionId,
+                    error: `Failed after ${MAX_RETRIES} attempts: ${errorMessage}`,
+                    clientNonce,
+                };
+            }
+            // Exponential backoff: 1s, 2s, 4s
+            await sleep(Math.pow(2, attempt - 1) * 1000);
+        }
+    }
+    // Should not reach here
+    return {
+        success: false,
+        evictionId: manifest.evictionId,
+        error: 'Unexpected failure',
+        clientNonce,
+    };
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// CONFIRM EVICTION (PHASE 2)
+// ═══════════════════════════════════════════════════════════════════════════
+/**
+ * Confirm eviction after local deletion.
+ * This is optional but recommended for audit completeness.
+ * Failure here is non-critical - data is already safe in R2.
+ */
+async function confirmEviction(apiUrl, authToken, evictionId, clientNonce, localDeletedCount) {
+    try {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), EVICTION_TIMEOUT_MS);
+        const response = await fetch(`${apiUrl}/api/v1/mcp/call`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${authToken}`,
+            },
+            body: JSON.stringify({
+                tool: 'ekkOS_EvictConfirm',
+                args: {
+                    evictionId,
+                    clientNonce,
+                    localDeletedCount,
+                },
+            }),
+            signal: controller.signal,
+        });
+        clearTimeout(timeoutId);
+        if (!response.ok) {
+            return {
+                success: false,
+                error: `HTTP ${response.status}`,
+            };
+        }
+        const data = await response.json();
+        if (!data.success) {
+            return {
+                success: false,
+                error: data.error || 'Confirm failed',
+            };
+        }
+        const result = data.result || data;
+        return {
+            success: true,
+            status: result.status,
+        };
+    }
+    catch (err) {
+        // Non-critical - data is already safe in R2
+        console.warn(`[EvictionClient] Confirm failed (non-critical): ${err instanceof Error ? err.message : err}`);
+        return {
+            success: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+/**
+ * Full handshake eviction - call this instead of fire-and-forget
+ *
+ * Returns:
+ * - success=true, status='prepared' → Safe to delete locally, then call confirmEviction
+ * - success=false → DO NOT delete locally
+ */
+async function handshakeEviction(messages, indices, estimatedTokens, reason, options) {
+    // Build manifest
+    const evictionId = createEvictionId(messages);
+    const fingerprints = messages.map(createFingerprint);
+    const manifest = {
+        evictionId,
+        messageIndices: indices,
+        fingerprints,
+        estimatedTokens,
+        evictionReason: reason,
+    };
+    // Check health first (fast fail)
+    const healthy = await checkEvictionHealth(options.apiUrl, options.authToken);
+    if (!healthy) {
+        console.warn('[EvictionClient] Proxy unavailable - handshake eviction skipped');
+        return {
+            success: false,
+            evictionId,
+            status: 'skipped',
+            error: 'Proxy unavailable',
+        };
+    }
+    // Phase 1: Prepare
+    const prepareResult = await prepareEviction(options.apiUrl, options.authToken, options.sessionId, options.sessionName, options.userId, options.tenantId, messages, manifest, options.projectPath);
+    if (!prepareResult.success) {
+        return {
+            success: false,
+            evictionId,
+            status: 'failed',
+            error: prepareResult.error,
+        };
+    }
+    return {
+        success: true,
+        evictionId: prepareResult.evictionId,
+        r2Key: prepareResult.r2Key,
+        bytesWritten: prepareResult.bytesWritten,
+        status: 'prepared',
+        clientNonce: prepareResult.clientNonce,
+    };
+}
+exports.default = {
+    prepareEviction,
+    confirmEviction,
+    handshakeEviction,
+    checkEvictionHealth,
+    createEvictionId,
+    createFingerprint,
+    queueEvictionForRetry,
+    drainRetryQueue,
+    getRetryQueueStats,
+};

package/dist/capture/index.d.ts

@@ -6,3 +6,5 @@
  */
 export * from './types';
 export * from './stream-tailer';
+export * from './jsonl-rewriter';
+export * from './transcript-repair';

package/dist/capture/index.js

@@ -22,3 +22,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./types"), exports);
 __exportStar(require("./stream-tailer"), exports);
+__exportStar(require("./jsonl-rewriter"), exports);
+__exportStar(require("./transcript-repair"), exports);

package/dist/capture/jsonl-rewriter.d.ts

@@ -0,0 +1,96 @@
+/**
+ * JSONL Sliding Window - Maximize Context for New Work
+ *
+ * Progressive eviction - always stay lean:
+ * - 60%+: trim junk (streaming chunks only)
+ * - 65%+: trim metadata (snapshots, system) + truncate tool_results
+ * - 70%+: trim tools (results, calls)
+ * - 80%+: emergency dump to 50%
+ *
+ * THINKING BLOCKS: Preserved (priority 6) - contain valuable reasoning
+ * Safety: Last 50 lines are NEVER evicted (recent work protection)
+ *
+ * PROXY MODE (EKKOS_PROXY_MODE=1):
+ * When API proxy is enabled, eviction happens at the API level before
+ * requests reach Anthropic. In this mode, JSONL rewriter only does
+ * junk cleanup (continuousClean) - no threshold-based eviction.
+ * This prevents duplicate eviction from two sources.
+ */
+import { type HandshakeEvictionResult } from './eviction-client.js';
+interface HandshakeEvictionContext {
+    sessionId: string;
+    sessionName: string;
+    userId?: string;
+    tenantId?: string;
+    projectPath?: string;
+}
+/**
+ * Result of the eviction flow (sync or async)
+ */
+export interface EvictionResult {
+    success: boolean;
+    evicted: number;
+    truncated: number;
+    newPercent: number;
+    handshakeUsed?: boolean;
+    evictionId?: string;
+    error?: string;
+}
+/**
+ * Evict messages using the Handshake Protocol (two-phase commit)
+ *
+ * CRITICAL: Returns success ONLY after R2 confirms backup.
+ * Caller must NOT delete locally until this returns success=true.
+ *
+ * @returns result with clientNonce for confirm phase, or error
+ */
+export declare function evictWithHandshake(lines: string[], indices: number[], context: HandshakeEvictionContext): Promise<HandshakeEvictionResult>;
+/**
+ * Complete the handshake by confirming local deletion
+ * Call this AFTER successfully deleting from local JSONL
+ */
+export declare function confirmLocalEviction(evictionId: string, clientNonce: string, deletedCount: number): Promise<boolean>;
+/**
+ * Check if handshake eviction is available (proxy reachable)
+ */
+export declare function isHandshakeEvictionAvailable(): Promise<boolean>;
+export declare function evictToTarget(filePath: string, currentPercent: number, sessionId?: string, sessionName?: string): {
+    success: boolean;
+    evicted: number;
+    truncated: number;
+    newPercent: number;
+};
+export declare function needsEviction(percent: number): boolean;
+/**
+ * Async eviction with proper two-phase commit handshake.
+ * Use this instead of evictToTarget when async context is available.
+ *
+ * @returns Promise<EvictionResult>
+ */
+export declare function evictToTargetAsync(filePath: string, currentPercent: number, sessionId?: string, sessionName?: string): Promise<EvictionResult>;
+/**
+ * Continuous clean - run every turn, remove junk regardless of threshold
+ * Removes: thinking blocks, streaming chunks, old file-history-snapshots
+ */
+export declare function continuousClean(filePath: string): {
+    cleaned: number;
+};
+export declare function emergencyEvict(filePath: string): {
+    success: boolean;
+    evicted: number;
+};
+/**
+ * Get eviction stats
+ */
+export declare function getEvictionStats(): {
+    entries: number;
+    lastEviction?: string;
+};
+/**
+ * Get evicted content for retrieval (used by ccDNA)
+ */
+export declare function getEvictedContent(limit?: number): Array<{
+    timestamp: string;
+    content: string[];
+}>;
+export {};