@aiplumber/session-recall 1.8.7 → 1.9.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiplumber/session-recall",
-  "version": "1.8.7",
+  "version": "1.9.2",
   "description": "Pull context from previous Claude Code sessions. Sessions end, context resets - this tool lets you continue where you left off.",
   "bin": {
     "session-recall": "./session-recall"

package/session-recall

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 
-const VERSION = '1.8.7';
+const VERSION = '1.9.2';
 
 const fs = require('fs');
 const path = require('path');
+const readline = require('readline');
 
 // Cross-platform home directory
 const HOME = process.env.HOME || process.env.USERPROFILE;
@@ -49,12 +50,7 @@ function readJsonl(filePath) {
     const stats = fs.statSync(filePath);
     const sizeMB = stats.size / (1024 * 1024);
 
-    if (sizeMB > LARGE_FILE_DANGER_MB && !process.env.SESSION_RECALL_FORCE) {
-      console.error(`[session-recall] ERROR: File is ${sizeMB.toFixed(0)}MB - too large, will crash.`);
-      console.error(`Tip: Use 'session-recall last 1 --compactions' for just current phase`);
-      console.error(`Or: SESSION_RECALL_FORCE=1 NODE_OPTIONS="--max-old-space-size=8192" session-recall ...`);
-      process.exit(1);
-    } else if (sizeMB > LARGE_FILE_WARNING_MB) {
+    if (sizeMB > LARGE_FILE_WARNING_MB) {
       console.error(`[session-recall] WARNING: ${sizeMB.toFixed(0)}MB file - may be slow`);
     }
   } catch (e) {
@@ -71,6 +67,129 @@ function readJsonl(filePath) {
   }).filter(Boolean);
 }
 
+// Streaming JSONL reader - processes line by line, filters noise, returns promise
+async function streamReadJsonl(filePath, opts = {}) {
+  return new Promise((resolve, reject) => {
+    const messages = [];
+    const rl = readline.createInterface({
+      input: fs.createReadStream(filePath, { encoding: 'utf-8' }),
+      crlfDelay: Infinity
+    });
+
+    rl.on('line', (line) => {
+      if (!line.trim()) return;
+      try {
+        const msg = JSON.parse(line);
+
+        // Skip noise types immediately - don't store
+        if (NOISE_TYPES.includes(msg.type)) return;
+
+        // If only scanning for compactions, check and skip non-markers
+        if (opts.compactionsOnly) {
+          if (isCompactionMarker(msg)) {
+            messages.push({
+              timestamp: msg.timestamp,
+              line: messages.length,
+              data: msg.data
+            });
+          }
+          return;
+        }
+
+        // Keep message for discourse extraction
+        messages.push(msg);
+      } catch (e) {
+        // Skip unparseable lines
+      }
+    });
+
+    rl.on('close', () => resolve(messages));
+    rl.on('error', reject);
+  });
+}
+
+// Lightweight scan for compaction markers only
+async function streamScanCompactions(filePath) {
+  return new Promise((resolve, reject) => {
+    const markers = [];
+    let lineNum = 0;
+
+    const rl = readline.createInterface({
+      input: fs.createReadStream(filePath, { encoding: 'utf-8' }),
+      crlfDelay: Infinity
+    });
+
+    rl.on('line', (line) => {
+      lineNum++;
+      // Quick string check before parsing - check both old and new formats
+      if (!line.includes('SessionStart:compact') && !line.includes('compact_boundary')) return;
+
+      try {
+        const msg = JSON.parse(line);
+        if (isCompactionMarker(msg)) {
+          markers.push({
+            line: lineNum,
+            timestamp: msg.timestamp,
+            timestampMs: msg.timestamp ? new Date(msg.timestamp).getTime() : 0
+          });
+        }
+      } catch (e) {}
+    });
+
+    rl.on('close', () => {
+      // Deduplicate markers
+      resolve(deduplicateCompactionMarkers(markers));
+    });
+    rl.on('error', reject);
+  });
+}
+
+// Extract deduplication logic from scanCompactionMarkers into reusable function
+function deduplicateCompactionMarkers(rawMarkers) {
+  if (rawMarkers.length === 0) return [];
+
+  // Deduplicate markers into compaction events
+  // Markers are same event if within COMPACTION_DEDUP_WINDOW_MS AND fewer than COMPACTION_DEDUP_MAX_MESSAGES between them
+  const compactions = [];
+  let currentCluster = [rawMarkers[0]];
+
+  for (let i = 1; i < rawMarkers.length; i++) {
+    const prev = currentCluster[currentCluster.length - 1];
+    const curr = rawMarkers[i];
+
+    const timeGapMs = curr.timestampMs - prev.timestampMs;
+
+    // For deduplication without full messages, just use time window
+    // (discourse counting needs full message bodies, which we don't have in lightweight scan)
+    if (timeGapMs <= COMPACTION_DEDUP_WINDOW_MS) {
+      currentCluster.push(curr);
+    } else {
+      // New cluster - finalize previous one
+      compactions.push(finalizeMarkerCluster(currentCluster));
+      currentCluster = [curr];
+    }
+  }
+
+  // Finalize last cluster
+  compactions.push(finalizeMarkerCluster(currentCluster));
+
+  return compactions;
+}
+
+// Convert a cluster of markers into a single compaction event (lightweight version)
+function finalizeMarkerCluster(cluster) {
+  const first = cluster[0];
+  const retries = cluster.length - 1;
+
+  return {
+    timestamp: first.timestamp,
+    timestampMs: first.timestampMs,
+    line: first.line,
+    retries: retries,
+    messagesAfter: 0  // Not computed in lightweight scan
+  };
+}
+
 function isToolResult(msg) {
   if (!msg.message?.content) return false;
   const content = msg.message.content;
@@ -1028,7 +1147,7 @@ function cwdToProjectFolder() {
   return null;
 }
 
-function cmdLast(arg1, arg2, opts) {
+async function cmdLast(arg1, arg2, opts) {
   // Usage: last [N] [folder]
   // arg1 could be: undefined, a number string, or a folder path
   // arg2 could be: undefined, or a folder path (if arg1 was a number)
@@ -1174,7 +1293,21 @@ function cmdLast(arg1, arg2, opts) {
   const candidateCount = Math.min(allSessions.length, n + 5);
   for (let i = 0; i < candidateCount; i++) {
     const s = allSessions[i];
-    s.msgs = readJsonl(s.filePath);
+    const stats = fs.statSync(s.filePath);
+    const sizeMB = stats.size / (1024 * 1024);
+
+    // If we need compaction markers (for --before-compaction or --compactions),
+    // scan for them FIRST before loading messages (since streaming filters them out)
+    if ((opts.beforeCompaction || opts.compactionPhases) && sizeMB > 100) {
+      s.compactionMarkers = await streamScanCompactions(s.filePath);
+    }
+
+    // Use streaming for large files, sync for small files
+    if (sizeMB > 100) {
+      s.msgs = await streamReadJsonl(s.filePath);
+    } else {
+      s.msgs = readJsonl(s.filePath);
+    }
     const timestamps = s.msgs.filter(m => m.timestamp).map(m => new Date(m.timestamp));
     s.lastTs = timestamps.length > 0 ? new Date(Math.max(...timestamps)) : s.startTs;
   }
@@ -1204,7 +1337,8 @@ function cmdLast(arg1, arg2, opts) {
   // Handle --compactions mode: show last N compaction phases instead of sessions
   if (opts.compactionPhases) {
     const session = selected[0];  // Use only the most recent session
-    const compactions = scanCompactionMarkers(session.msgs);
+    // Use pre-scanned markers if available (for large files), otherwise scan now
+    const compactions = session.compactionMarkers || scanCompactionMarkers(session.msgs);
 
     if (compactions.length === 0) {
       console.log('[session-recall v' + VERSION + '] No compaction events found - showing full session as single phase');
@@ -1419,7 +1553,8 @@ function cmdLast(arg1, arg2, opts) {
     }
     // Apply before-compaction filter for dry-run
     if (opts.beforeCompaction && selected.length > 0) {
-      const compactions = scanCompactionMarkers(selected[0].msgs);
+      // Use pre-scanned markers if available (for large files), otherwise scan now
+      const compactions = selected[0].compactionMarkers || scanCompactionMarkers(selected[0].msgs);
       if (compactions.length > 0) {
         const n = opts.beforeCompactionN || compactions.length;
         if (n < 1 || n > compactions.length) {
@@ -1501,7 +1636,8 @@ function cmdLast(arg1, arg2, opts) {
 
     // Filter to only show messages BEFORE the selected compaction boundary
     if (opts.beforeCompaction) {
-      const compactions = scanCompactionMarkers(session.msgs);
+      // Use pre-scanned markers if available (for large files), otherwise scan now
+      const compactions = session.compactionMarkers || scanCompactionMarkers(session.msgs);
       if (compactions.length > 0) {
         const n = opts.beforeCompactionN || compactions.length;
         if (n < 1 || n > compactions.length) {
@@ -2268,12 +2404,19 @@ function scanSessionTangents(filePath) {
 // ========== COMPACTION DETECTION ==========
 
 // Check if a message is a compaction marker
+// Supports both old format (progress + hook) and new format (system + subtype)
 function isCompactionMarker(msg) {
-  if (msg.type !== 'progress') return false;
-  if (!msg.data) return false;
-  return msg.data.type === 'hook_progress' &&
-         msg.data.hookEvent === 'SessionStart' &&
-         msg.data.hookName === 'SessionStart:compact';
+  // New format (Claude Code 2.1+): system message with compact_boundary subtype
+  if (msg.type === 'system' && msg.subtype === 'compact_boundary') {
+    return true;
+  }
+  // Old format: progress message with hook
+  if (msg.type === 'progress' && msg.data) {
+    return msg.data.type === 'hook_progress' &&
+           msg.data.hookEvent === 'SessionStart' &&
+           msg.data.hookName === 'SessionStart:compact';
+  }
+  return false;
 }
 
 // Scan session for compaction markers with deduplication
@@ -2357,7 +2500,7 @@ function finalizeCluster(cluster, messages) {
 }
 
 // cmdCompactions: List compaction events in a JSONL file
-function cmdCompactions(jsonlPath, opts) {
+async function cmdCompactions(jsonlPath, opts) {
   // Auto-detect if no path provided
   if (!jsonlPath) {
     const dir = cwdToProjectFolder();
@@ -2382,31 +2525,45 @@ function cmdCompactions(jsonlPath, opts) {
     jsonlPath = filesWithStats[0].filePath;
   }
 
-  const messages = readJsonl(jsonlPath);
-  const compactions = scanCompactionMarkers(messages);
+  const compactions = await streamScanCompactions(jsonlPath);
+  // Read full messages only if we found compactions and need phase token estimates
+  let messages = [];
+  if (compactions.length > 0) {
+    const stats = fs.statSync(jsonlPath);
+    const sizeMB = stats.size / (1024 * 1024);
+    if (sizeMB > 100) {
+      messages = await streamReadJsonl(jsonlPath);
+    } else {
+      messages = readJsonl(jsonlPath);
+    }
+  }
 
   if (compactions.length === 0) {
     console.log('No compaction events found.');
     return;
   }
 
-  // Calculate token estimates for each phase
-  function estimatePhaseTokens(startLine, endLine) {
+  // Calculate token estimates for messages between two timestamps
+  function estimatePhaseTokensByTime(startTs, endTs) {
     let tokens = 0;
-    for (let j = startLine; j < endLine && j < messages.length; j++) {
-      const msg = messages[j];
-      if (msg.message?.content) {
-        const content = msg.message.content;
-        if (typeof content === 'string') {
-          tokens += estimateTokens({ length: content.length });
-        } else if (Array.isArray(content)) {
-          for (const block of content) {
-            if (block.text) tokens += estimateTokens({ length: block.text.length });
+    let count = 0;
+    for (const msg of messages) {
+      const msgTs = msg.timestamp ? new Date(msg.timestamp).getTime() : 0;
+      if (msgTs >= startTs && msgTs < endTs) {
+        count++;
+        if (msg.message?.content) {
+          const content = msg.message.content;
+          if (typeof content === 'string') {
+            tokens += estimateTokens({ length: content.length });
+          } else if (Array.isArray(content)) {
+            for (const block of content) {
+              if (block.text) tokens += estimateTokens({ length: block.text.length });
+            }
           }
         }
       }
     }
-    return tokens;
+    return { tokens, count };
   }
 
   // Get file size
@@ -2426,14 +2583,14 @@ function cmdCompactions(jsonlPath, opts) {
     const c = compactions[i];
     const ts = c.timestamp ? new Date(c.timestamp).toISOString().substring(11, 19) : '??:??:??';
 
-    // Calculate phase boundaries
-    const startLine = i === 0 ? 0 : compactions[i - 1].line;
-    const endLine = c.line;
-    const phaseTokens = estimatePhaseTokens(startLine, endLine);
+    // Calculate phase boundaries using timestamps
+    const startTs = i === 0 ? 0 : compactions[i - 1].timestampMs;
+    const endTs = c.timestampMs;
+    const { tokens: phaseTokens, count: phaseCount } = estimatePhaseTokensByTime(startTs, endTs);
     const tokensK = (phaseTokens / 1000).toFixed(0) + 'k';
 
     const num = String(i + 1).padStart(2);
-    const msgs = String(endLine - startLine).padStart(5);
+    const msgs = String(phaseCount).padStart(5);
     const toks = tokensK.padStart(8);
 
     console.log(`${num}   ${ts}  ${msgs}  ${toks}   Phase ${i + 1}`);
@@ -2441,11 +2598,10 @@ function cmdCompactions(jsonlPath, opts) {
 
   // Current phase (after last compaction)
   const lastCompaction = compactions[compactions.length - 1];
-  const currentMsgs = messages.length - lastCompaction.line;
-  const currentTokens = estimatePhaseTokens(lastCompaction.line, messages.length);
+  const { tokens: currentTokens, count: currentCount } = estimatePhaseTokensByTime(lastCompaction.timestampMs, Date.now());
   const currentK = (currentTokens / 1000).toFixed(0) + 'k';
   console.log(`───────────────────────────────────────`);
-  console.log(`     (current)  ${String(currentMsgs).padStart(5)}  ${currentK.padStart(8)}   After #${compactions.length}`);
+  console.log(`     (current)  ${String(currentCount).padStart(5)}  ${currentK.padStart(8)}   After #${compactions.length}`);
   console.log(``);
 
   // Usage hint
@@ -2660,6 +2816,7 @@ function parseArgs(args) {
 
 // ========== MAIN ==========
 
+(async () => {
 const args = process.argv.slice(2);
 const { opts, positional } = parseArgs(args);
 
@@ -2836,7 +2993,7 @@ switch (command) {
     cmdCost(jsonlPath);
     break;
   case 'last':
-    cmdLast(jsonlPath, positional[2], opts);
+    await cmdLast(jsonlPath, positional[2], opts);
     break;
   case 'tools':
     cmdTools(opts);
@@ -2845,7 +3002,7 @@ switch (command) {
     cmdCheckpoints(opts);
     break;
   case 'compactions':
-    cmdCompactions(jsonlPath, opts);
+    await cmdCompactions(jsonlPath, opts);
     break;
   case 'help':
     showHelp();
@@ -2855,3 +3012,4 @@ switch (command) {
     showHelp();
     process.exit(1);
 }
+})();