@stackmemoryai/stackmemory 0.8.1 → 1.0.1

package/scripts/measure-handoff-impact.mjs

@@ -1,395 +0,0 @@
-#!/usr/bin/env node
-/**
- * Measure actual handoff context impact with real data
- * Validates claims about token savings
- */
-
-import { readFileSync, existsSync, readdirSync, statSync } from 'fs';
-import { join } from 'path';
-import { homedir } from 'os';
-import Database from 'better-sqlite3';
-
-// Token estimation: Claude uses ~3.5-4 chars per token
-function estimateTokens(text) {
-  return Math.ceil(text.length / 4);
-}
-
-// More accurate estimation considering code vs prose
-function estimateTokensAccurate(text) {
-  if (!text || typeof text !== 'string') return 0;
-  const baseEstimate = text.length / 3.5;
-
-  // Check if code-heavy (more tokens per char)
-  const codeIndicators = (text.match(/[{}\[\]();=]/g) || []).length;
-  const codeScore = codeIndicators / Math.max(text.length, 1) * 100;
-
-  if (codeScore > 5) {
-    return Math.ceil(baseEstimate * 1.2);
-  }
-  return Math.ceil(baseEstimate);
-}
-
-function measureHandoffs() {
-  const handoffPath = join(homedir(), '.stackmemory', 'context.db');
-  const metrics = [];
-
-  if (!existsSync(handoffPath)) {
-    console.log('   No context.db found at', handoffPath);
-    return metrics;
-  }
-
-  try {
-    const db = new Database(handoffPath, { readonly: true });
-
-    // Check if handoff_requests table exists
-    const tableCheck = db.prepare(`
-      SELECT name FROM sqlite_master
-      WHERE type='table' AND name='handoff_requests'
-    `).get();
-
-    if (!tableCheck) {
-      console.log('   No handoff_requests table found');
-      db.close();
-      return metrics;
-    }
-
-    const handoffs = db.prepare(`
-      SELECT id, message, created_at
-      FROM handoff_requests
-      ORDER BY created_at DESC
-      LIMIT 10
-    `).all();
-
-    for (const h of handoffs) {
-      const message = h.message || '';
-      metrics.push({
-        handoffId: h.id,
-        handoffChars: message.length,
-        handoffTokens: estimateTokensAccurate(message),
-        createdAt: new Date(h.created_at).toISOString(),
-      });
-    }
-
-    db.close();
-  } catch (err) {
-    console.log('   Error reading handoffs:', err.message);
-  }
-
-  return metrics;
-}
-
-function measureLastHandoffFile() {
-  const paths = [
-    join(process.cwd(), '.stackmemory', 'last-handoff.md'),
-    join(homedir(), '.stackmemory', 'last-handoff.md'),
-  ];
-
-  for (const handoffPath of paths) {
-    if (existsSync(handoffPath)) {
-      const content = readFileSync(handoffPath, 'utf-8');
-      return {
-        source: handoffPath,
-        charCount: content.length,
-        estimatedTokens: estimateTokensAccurate(content),
-        lineCount: content.split('\n').length,
-      };
-    }
-  }
-  return null;
-}
-
-function measureClaudeConversations() {
-  const claudeProjectsDir = join(homedir(), '.claude', 'projects');
-  const metrics = [];
-
-  if (!existsSync(claudeProjectsDir)) {
-    return metrics;
-  }
-
-  try {
-    const projectDirs = readdirSync(claudeProjectsDir);
-
-    for (const dir of projectDirs.slice(0, 5)) {
-      const projectPath = join(claudeProjectsDir, dir);
-
-      try {
-        const stat = statSync(projectPath);
-        if (!stat.isDirectory()) continue;
-
-        const files = readdirSync(projectPath).filter(f => f.endsWith('.jsonl'));
-
-        for (const file of files.slice(0, 3)) {
-          const filePath = join(projectPath, file);
-          try {
-            const content = readFileSync(filePath, 'utf-8');
-            metrics.push({
-              source: `${dir.slice(0, 20)}.../${file.slice(0, 12)}...`,
-              charCount: content.length,
-              estimatedTokens: estimateTokensAccurate(content),
-              lineCount: content.split('\n').length,
-            });
-          } catch {
-            // Skip unreadable files
-          }
-        }
-      } catch {
-        // Skip inaccessible directories
-      }
-    }
-  } catch {
-    // Directory listing failed
-  }
-
-  return metrics;
-}
-
-function measureFramesAndEvents() {
-  const dbPath = join(homedir(), '.stackmemory', 'context.db');
-
-  if (!existsSync(dbPath)) {
-    return null;
-  }
-
-  try {
-    const db = new Database(dbPath, { readonly: true });
-
-    // Get frame count and content size
-    let frameResult = { count: 0, totalChars: 0 };
-    try {
-      frameResult = db.prepare(`
-        SELECT COUNT(*) as count,
-               COALESCE(SUM(LENGTH(COALESCE(name, '') || COALESCE(json(inputs), '{}') || COALESCE(json(outputs), '{}'))), 0) as totalChars
-        FROM frames
-      `).get() || { count: 0, totalChars: 0 };
-    } catch {
-      // Table might not exist
-    }
-
-    // Get event count and content size
-    let eventResult = { count: 0, totalChars: 0 };
-    try {
-      eventResult = db.prepare(`
-        SELECT COUNT(*) as count,
-               COALESCE(SUM(LENGTH(COALESCE(event_type, '') || COALESCE(json(payload), '{}'))), 0) as totalChars
-        FROM events
-      `).get() || { count: 0, totalChars: 0 };
-    } catch {
-      // Table might not exist
-    }
-
-    db.close();
-
-    const totalChars = (frameResult.totalChars || 0) + (eventResult.totalChars || 0);
-
-    return {
-      sessionId: 'aggregate',
-      frameCount: frameResult.count || 0,
-      eventCount: eventResult.count || 0,
-      totalChars: totalChars,
-      estimatedSessionTokens: estimateTokensAccurate('x'.repeat(Math.min(totalChars, 100000))),
-    };
-  } catch (err) {
-    console.log('   Error measuring frames/events:', err.message);
-    return null;
-  }
-}
-
-function formatNumber(n) {
-  if (n >= 1000000) {
-    return (n / 1000000).toFixed(1) + 'M';
-  }
-  if (n >= 1000) {
-    return (n / 1000).toFixed(1) + 'K';
-  }
-  return n.toString();
-}
-
-async function main() {
-  console.log('========================================');
-  console.log('   HANDOFF CONTEXT IMPACT ANALYSIS');
-  console.log('   (Actual Measurements)');
-  console.log('========================================\n');
-
-  // 1. Measure last handoff file
-  console.log('1. LAST HANDOFF FILE');
-  console.log('--------------------');
-  const lastHandoff = measureLastHandoffFile();
-  if (lastHandoff) {
-    console.log(`   Source: ${lastHandoff.source}`);
-    console.log(`   Characters: ${formatNumber(lastHandoff.charCount)}`);
-    console.log(`   Lines: ${lastHandoff.lineCount}`);
-    console.log(`   Estimated tokens: ${formatNumber(lastHandoff.estimatedTokens)}`);
-  } else {
-    console.log('   No handoff file found');
-  }
-  console.log('');
-
-  // 2. Measure handoffs from database
-  console.log('2. HANDOFFS FROM DATABASE');
-  console.log('-------------------------');
-  const handoffs = measureHandoffs();
-  if (handoffs.length > 0) {
-    let totalTokens = 0;
-    for (const h of handoffs) {
-      console.log(`   ${h.handoffId.slice(0, 8)}: ${formatNumber(h.handoffTokens)} tokens (${formatNumber(h.handoffChars)} chars)`);
-      totalTokens += h.handoffTokens;
-    }
-    const avgTokens = Math.round(totalTokens / handoffs.length);
-    console.log(`   Average: ${formatNumber(avgTokens)} tokens per handoff`);
-  } else {
-    console.log('   No handoffs in database');
-  }
-  console.log('');
-
-  // 3. Measure Claude conversation files
-  console.log('3. CLAUDE CONVERSATION FILES');
-  console.log('----------------------------');
-  const conversations = measureClaudeConversations();
-  if (conversations.length > 0) {
-    let totalConvTokens = 0;
-    let maxConvTokens = 0;
-    for (const c of conversations) {
-      console.log(`   ${c.source}: ${formatNumber(c.estimatedTokens)} tokens (${formatNumber(c.charCount)} chars)`);
-      totalConvTokens += c.estimatedTokens;
-      maxConvTokens = Math.max(maxConvTokens, c.estimatedTokens);
-    }
-    const avgConvTokens = Math.round(totalConvTokens / conversations.length);
-    console.log(`   Average: ${formatNumber(avgConvTokens)} tokens per conversation`);
-    console.log(`   Max: ${formatNumber(maxConvTokens)} tokens`);
-  } else {
-    console.log('   No conversation files found');
-  }
-  console.log('');
-
-  // 4. Measure StackMemory database
-  console.log('4. STACKMEMORY DATABASE CONTENT');
-  console.log('-------------------------------');
-  const dbMetrics = measureFramesAndEvents();
-  if (dbMetrics) {
-    console.log(`   Frames: ${dbMetrics.frameCount}`);
-    console.log(`   Events: ${dbMetrics.eventCount}`);
-    console.log(`   Total chars stored: ${formatNumber(dbMetrics.totalChars)}`);
-    console.log(`   Estimated tokens: ~${formatNumber(dbMetrics.estimatedSessionTokens)}`);
-  } else {
-    console.log('   No database metrics available');
-  }
-  console.log('');
-
-  // 5. Calculate compression ratios
-  console.log('5. COMPRESSION ANALYSIS');
-  console.log('-----------------------');
-
-  const avgHandoffTokens = handoffs.length > 0
-    ? Math.round(handoffs.reduce((sum, h) => sum + h.handoffTokens, 0) / handoffs.length)
-    : (lastHandoff?.estimatedTokens || 2000);
-
-  const avgConversationTokens = conversations.length > 0
-    ? Math.round(conversations.reduce((sum, c) => sum + c.estimatedTokens, 0) / conversations.length)
-    : 80000;
-
-  // Typical session sizes based on actual data
-  const sessionSizes = {
-    'short (2h)': 35000,
-    'medium (4h)': 78000,
-    'long (8h)': 142000,
-    'measured avg': avgConversationTokens,
-  };
-
-  console.log('\n   Compression Ratios (using actual handoff size):');
-  console.log(`   Handoff size: ${formatNumber(avgHandoffTokens)} tokens\n`);
-
-  for (const [label, size] of Object.entries(sessionSizes)) {
-    const reduction = ((size - avgHandoffTokens) / size * 100).toFixed(1);
-    const saved = size - avgHandoffTokens;
-    console.log(`   ${label.padEnd(14)}: ${formatNumber(size).padStart(6)} -> ${formatNumber(avgHandoffTokens).padStart(5)} = ${reduction.padStart(5)}% reduction (${formatNumber(saved)} saved)`);
-  }
-
-  console.log('');
-
-  // 6. Context window impact
-  console.log('6. CONTEXT WINDOW IMPACT');
-  console.log('------------------------');
-  const contextWindow = 200000;
-  const systemPrompt = 2000;
-  const currentTools = 10000;
-
-  const withoutHandoff = {
-    used: systemPrompt + avgConversationTokens + currentTools,
-  };
-  withoutHandoff.available = Math.max(0, contextWindow - withoutHandoff.used);
-
-  const withHandoff = {
-    used: systemPrompt + avgHandoffTokens + currentTools,
-  };
-  withHandoff.available = contextWindow - withHandoff.used;
-
-  console.log(`   Context window: ${formatNumber(contextWindow)} tokens`);
-  console.log(`   System prompt: ${formatNumber(systemPrompt)} tokens`);
-  console.log(`   Current tools: ${formatNumber(currentTools)} tokens\n`);
-
-  console.log('   WITHOUT HANDOFF:');
-  console.log(`     Conversation history: ${formatNumber(avgConversationTokens)} tokens`);
-  console.log(`     Total used: ${formatNumber(withoutHandoff.used)} tokens`);
-  console.log(`     Available for work: ${formatNumber(withoutHandoff.available)} tokens (${(withoutHandoff.available / contextWindow * 100).toFixed(1)}%)`);
-  console.log('');
-
-  console.log('   WITH HANDOFF:');
-  console.log(`     Handoff summary: ${formatNumber(avgHandoffTokens)} tokens`);
-  console.log(`     Total used: ${formatNumber(withHandoff.used)} tokens`);
-  console.log(`     Available for work: ${formatNumber(withHandoff.available)} tokens (${(withHandoff.available / contextWindow * 100).toFixed(1)}%)`);
-  console.log('');
-
-  const improvement = withHandoff.available - withoutHandoff.available;
-  const improvementPct = withoutHandoff.available > 0
-    ? (improvement / withoutHandoff.available * 100).toFixed(1)
-    : 'N/A';
-  console.log(`   IMPROVEMENT: +${formatNumber(improvement)} tokens (+${improvementPct}% more capacity)`);
-
-  console.log('\n========================================');
-  console.log('   SUMMARY & CLAIM VALIDATION');
-  console.log('========================================\n');
-
-  const actualReduction = ((avgConversationTokens - avgHandoffTokens) / avgConversationTokens * 100).toFixed(1);
-
-  console.log(`   Measured handoff size: ${formatNumber(avgHandoffTokens)} tokens`);
-  console.log(`   Measured conversation size: ${formatNumber(avgConversationTokens)} tokens`);
-  console.log(`   Measured compression: ${actualReduction}%`);
-  console.log(`   Measured context freed: ${formatNumber(improvement)} tokens`);
-  console.log('');
-
-  // Validate claims from document
-  console.log('   CLAIM VALIDATION:');
-  console.log('   -----------------');
-
-  const claims = [
-    {
-      name: 'Reduction range',
-      claimed: '85-98%',
-      measured: `${actualReduction}%`,
-      valid: parseFloat(actualReduction) >= 85 && parseFloat(actualReduction) <= 98,
-    },
-    {
-      name: 'Handoff size',
-      claimed: '1K-5K tokens',
-      measured: `${formatNumber(avgHandoffTokens)} tokens`,
-      valid: avgHandoffTokens >= 1000 && avgHandoffTokens <= 5000,
-    },
-    {
-      name: 'Conversation size',
-      claimed: '50K-150K tokens',
-      measured: `${formatNumber(avgConversationTokens)} tokens`,
-      valid: avgConversationTokens >= 50000 && avgConversationTokens <= 150000,
-    },
-  ];
-
-  for (const claim of claims) {
-    const status = claim.valid ? 'VALID' : 'REVISE';
-    console.log(`   ${claim.name}:`);
-    console.log(`     Claimed: ${claim.claimed}`);
-    console.log(`     Measured: ${claim.measured}`);
-    console.log(`     Status: ${status}`);
-    console.log('');
-  }
-}
-
-main().catch(console.error);