@equilateral_ai/mindmeld 3.4.0 → 3.5.1

package/hooks/pre-compact.js

@@ -18,6 +18,56 @@ const path = require('path');
 const fs = require('fs').promises;
 const crypto = require('crypto');
 
+/**
+ * Scrub sensitive data from text before sending to MindMeld API.
+ * Replaces AWS keys, API tokens, passwords, connection strings,
+ * private keys, and generic secrets with [REDACTED].
+ * @param {string} text - Text that may contain secrets
+ * @returns {string} Text with secrets replaced by [REDACTED]
+ */
+function scrubSecrets(text) {
+  if (typeof text !== 'string') return text;
+
+  const patterns = [
+    // AWS Access Keys
+    /AKIA[0-9A-Z]{16}/g,
+    // AWS Secret Keys
+    /(?:aws_secret_access_key|secret_key|secretAccessKey)\s*[=:]\s*['"]?[A-Za-z0-9/+=]{40}['"]?/gi,
+    // Generic API tokens
+    /(?:api[_-]?key|api[_-]?token|auth[_-]?token|bearer)\s*[=:]\s*['"]?[A-Za-z0-9_\-\.]{20,}['"]?/gi,
+    // Passwords
+    /(?:password|passwd|pwd)\s*[=:]\s*['"]?[^\s'"]{4,}['"]?/gi,
+    // Connection strings
+    /(?:postgres|mysql|mongodb|redis):\/\/[^\s'"]+/gi,
+    // Private keys
+    /-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----[\s\S]*?-----END\s+(?:RSA\s+)?PRIVATE\s+KEY-----/g,
+    // Generic secrets
+    /(?:secret|token|credential)\s*[=:]\s*['"]?[A-Za-z0-9_\-\.\/+=]{16,}['"]?/gi,
+  ];
+
+  let scrubbed = text;
+  for (const pattern of patterns) {
+    scrubbed = scrubbed.replace(pattern, '[REDACTED]');
+  }
+  return scrubbed;
+}
+
+/**
+ * Deep-scrub secrets from an object by traversing all string values.
+ * @param {*} obj - Object, array, or primitive to scrub
+ * @returns {*} Scrubbed copy (original is not mutated)
+ */
+function scrubSecretsDeep(obj) {
+  if (typeof obj === 'string') return scrubSecrets(obj);
+  if (obj === null || obj === undefined || typeof obj !== 'object') return obj;
+  if (Array.isArray(obj)) return obj.map(scrubSecretsDeep);
+  const result = {};
+  for (const [key, value] of Object.entries(obj)) {
+    result[key] = scrubSecretsDeep(value);
+  }
+  return result;
+}
+
 // LLM Pattern Detection (optional - requires ANTHROPIC_API_KEY)
 let LLMPatternDetector = null;
 try {
@@ -200,7 +250,7 @@ async function harvestPatterns(sessionTranscript) {
         ? sessionTranscript
         : sessionTranscript.transcript || JSON.stringify(sessionTranscript);
 
-      llmAnalysis = await detector.analyzeSessionTranscript(transcriptText, {
+      llmAnalysis = await detector.analyzeSessionTranscript(scrubSecrets(transcriptText), {
         projectName: path.basename(process.cwd()),
         filesChanged: sessionTranscript.filesChanged || []
       });
@@ -238,23 +288,23 @@ async function harvestPatterns(sessionTranscript) {
     // 2. Validate against standards
     const validationResults = await validatePatterns(mindmeld, patterns);
 
-    // 3. Record violations
+    // 3. Record violations (scrub secrets before sending to API)
     for (const result of validationResults.violations) {
-      await mindmeld.recordViolation({
+      await mindmeld.recordViolation(scrubSecretsDeep({
         pattern: result.pattern,
         violations: result.violations,
         sessionId: sessionId,
         userId: userId
-      });
+      }));
     }
 
-    // 4. Reinforce valid patterns
+    // 4. Reinforce valid patterns (scrub secrets before sending to API)
     for (const result of validationResults.valid) {
-      await mindmeld.reinforcePattern({
+      await mindmeld.reinforcePattern(scrubSecretsDeep({
         pattern: result.pattern,
         sessionId: sessionId,
         userId: userId
-      });
+      }));
     }
 
     // 5. Check for promotion candidates
@@ -274,7 +324,26 @@ async function harvestPatterns(sessionTranscript) {
       console.error('[MindMeld] Plan harvesting failed (non-fatal):', error.message);
     }
 
-    // 7. Log results
+    // 7. Detect and send corrections from conversation text
+    let correctionsDetected = 0;
+    try {
+      const transcriptText = typeof sessionTranscript === 'string'
+        ? sessionTranscript
+        : sessionTranscript.transcript || JSON.stringify(sessionTranscript);
+
+      const corrections = detectCorrections(transcriptText);
+      correctionsDetected = corrections.length;
+
+      if (corrections.length > 0) {
+        console.error(`[MindMeld] Detected ${corrections.length} correction(s) in session`);
+        await sendCorrections(corrections, authToken, apiConfig);
+        console.error(`[MindMeld] Sent ${corrections.length} correction(s) to API`);
+      }
+    } catch (error) {
+      console.error('[MindMeld] Correction harvesting failed (non-fatal):', error.message);
+    }
+
+    // 8. Log results
     const elapsed = Date.now() - startTime;
     const summary = {
       patternsDetected: patterns.length,
@@ -282,6 +351,7 @@ async function harvestPatterns(sessionTranscript) {
       reinforced: validationResults.valid.length,
       promotionCandidates: candidates.length,
       plansHarvested: harvestedPlans.length,
+      correctionsDetected: correctionsDetected,
       plans: harvestedPlans,
       readmeStale: null,
       readmeUpdateRecommended: false,
@@ -436,10 +506,10 @@ function parsePlanFile(filename, content, stat) {
     sizeBytes: stat.size,
     lineCount: lines.length,
     sections: Object.keys(sections),
-    context: sections['context'] || null,
+    context: scrubSecrets(sections['context'] || null),
     filesReferenced: fileRefs.slice(0, 20),
     projectHints: projectHints,
-    content: content
+    content: scrubSecrets(content)
   };
 }
 
@@ -502,6 +572,109 @@ async function checkPromotionCandidates(mindmeld, validPatterns) {
   return candidates;
 }
 
+/**
+ * Detect correction language patterns in conversation text.
+ * Scans for phrases indicating the user corrected the AI's approach.
+ * @param {string} conversationText - Raw conversation text to scan
+ * @returns {Array<{correction_text: string, context_before: string, context_after: string, pattern_matched: string}>}
+ */
+function detectCorrections(conversationText) {
+  if (typeof conversationText !== 'string' || conversationText.length === 0) return [];
+
+  const correctionPatterns = [
+    /no,? don'?t/gi,
+    /that'?s wrong/gi,
+    /instead,? do/gi,
+    /not like that/gi,
+    /revert that/gi,
+    /undo that/gi,
+    /shouldn'?t have/gi,
+    /wrong approach/gi,
+    /bad pattern/gi,
+    /don'?t use/gi,
+    /never do that/gi,
+    /stop doing/gi,
+  ];
+
+  const corrections = [];
+
+  for (const pattern of correctionPatterns) {
+    let match;
+    while ((match = pattern.exec(conversationText)) !== null) {
+      const matchStart = match.index;
+      const matchEnd = matchStart + match[0].length;
+
+      const contextStart = Math.max(0, matchStart - 100);
+      const contextEnd = Math.min(conversationText.length, matchEnd + 100);
+
+      corrections.push({
+        correction_text: match[0],
+        context_before: conversationText.slice(contextStart, matchStart),
+        context_after: conversationText.slice(matchEnd, contextEnd),
+        pattern_matched: pattern.source,
+      });
+    }
+  }
+
+  return corrections;
+}
+
+/**
+ * Send detected corrections to the MindMeld API.
+ * @param {Array} corrections - Array of correction objects from detectCorrections
+ * @param {string} authToken - Auth token for API calls
+ * @param {{apiUrl: string}} apiConfig - API configuration
+ * @returns {Promise<void>}
+ */
+async function sendCorrections(corrections, authToken, apiConfig) {
+  if (!corrections || corrections.length === 0) return;
+
+  const url = `${apiConfig.apiUrl}/corrections`;
+  const body = JSON.stringify({
+    corrections: scrubSecretsDeep(corrections),
+    source: 'hook-harvest',
+  });
+
+  const https = require('https');
+  const http = require('http');
+  const parsedUrl = new URL(url);
+  const transport = parsedUrl.protocol === 'https:' ? https : http;
+
+  const headers = {
+    'Content-Type': 'application/json',
+    'Content-Length': Buffer.byteLength(body),
+  };
+  if (authToken) {
+    headers['Authorization'] = `Bearer ${authToken}`;
+  }
+
+  return new Promise((resolve, reject) => {
+    const req = transport.request(
+      {
+        hostname: parsedUrl.hostname,
+        port: parsedUrl.port,
+        path: parsedUrl.pathname,
+        method: 'POST',
+        headers,
+      },
+      (res) => {
+        let data = '';
+        res.on('data', (chunk) => (data += chunk));
+        res.on('end', () => {
+          if (res.statusCode >= 200 && res.statusCode < 300) {
+            resolve(data);
+          } else {
+            reject(new Error(`Corrections API returned ${res.statusCode}: ${data}`));
+          }
+        });
+      }
+    );
+    req.on('error', reject);
+    req.write(body);
+    req.end();
+  });
+}
+
 /**
  * Generate session ID using crypto for consistency
  */
@@ -650,20 +823,95 @@ async function generatePostCompactContext(summary, llmAnalysis) {
   return sections.join('\n');
 }
 
-// Execute if called directly
-if (require.main === module) {
-  // Read session transcript from stdin or args
-  const input = process.argv[2];
+/**
+ * Read stdin with timeout.
+ * Claude Code hooks receive JSON input via stdin, not command-line arguments.
+ * @returns {Promise<string>} stdin content or empty string
+ */
+function readStdin() {
+  return new Promise((resolve) => {
+    let data = '';
+    const timeout = setTimeout(() => resolve(data), 2000);
+
+    if (process.stdin.isTTY) {
+      clearTimeout(timeout);
+      resolve('');
+      return;
+    }
+
+    process.stdin.setEncoding('utf-8');
+    process.stdin.on('data', chunk => { data += chunk; });
+    process.stdin.on('end', () => {
+      clearTimeout(timeout);
+      resolve(data);
+    });
+    process.stdin.on('error', () => {
+      clearTimeout(timeout);
+      resolve('');
+    });
+    process.stdin.resume();
+  });
+}
+
+/**
+ * Read the session transcript from the best available source.
+ * Priority:
+ *   1. stdin JSON with transcript_path (Claude Code hooks protocol)
+ *   2. process.argv[2] (legacy / manual testing, limited by OS ARG_MAX)
+ * @returns {Promise<Object>} Parsed session transcript
+ */
+async function readTranscriptInput() {
+  // 1. Try stdin — Claude Code hooks pass JSON with transcript_path
+  try {
+    const stdin = await readStdin();
+    if (stdin) {
+      const hookInput = JSON.parse(stdin);
+
+      // If we got a transcript_path, read the file (no ARG_MAX limit)
+      if (hookInput.transcript_path) {
+        console.error(`[MindMeld] Reading transcript from file: ${hookInput.transcript_path}`);
+        const content = await fs.readFile(hookInput.transcript_path, 'utf-8');
+        const transcript = parseSessionTranscript(content);
+        // Merge hook metadata (session_id, cwd) into transcript
+        if (hookInput.session_id && !transcript.sessionId) {
+          transcript.sessionId = hookInput.session_id;
+        }
+        return transcript;
+      }
+
+      // If stdin itself contains the transcript (inline JSON)
+      if (hookInput.transcript || hookInput.messages || hookInput.sessionId) {
+        return parseSessionTranscript(JSON.stringify(hookInput));
+      }
 
-  if (!input) {
-    console.error('[MindMeld] Usage: pre-compact.js <session-transcript-json>');
-    process.exit(0);
+      // Stdin had JSON but no transcript — try parsing it as-is
+      return parseSessionTranscript(stdin);
+    }
+  } catch (e) {
+    console.error(`[MindMeld] stdin parse failed (falling back to argv): ${e.message}`);
+  }
+
+  // 2. Fallback: process.argv[2] (legacy, limited to ~1MB on macOS)
+  const argInput = process.argv[2];
+  if (argInput) {
+    console.error('[MindMeld] Reading transcript from argv (legacy mode)');
+    return parseSessionTranscript(argInput);
   }
 
-  const sessionTranscript = parseSessionTranscript(input);
+  return null;
+}
+
+// Execute if called directly
+if (require.main === module) {
+  readTranscriptInput()
+    .then(async (sessionTranscript) => {
+      if (!sessionTranscript) {
+        console.error('[MindMeld] No transcript input received (stdin or argv)');
+        process.exit(0);
+      }
+
+      const result = await harvestPatterns(sessionTranscript);
 
-  harvestPatterns(sessionTranscript)
-    .then(async (result) => {
       // Generate and output context summary for post-compaction injection
       const postCompactContext = await generatePostCompactContext(
         result,
@@ -680,4 +928,4 @@ if (require.main === module) {
     });
 }
 
-module.exports = { harvestPatterns, harvestPlans, parseSessionTranscript, generatePostCompactContext };
+module.exports = { harvestPatterns, harvestPlans, parseSessionTranscript, generatePostCompactContext, scrubSecrets, scrubSecretsDeep, detectCorrections, sendCorrections, readStdin, readTranscriptInput };

package/hooks/session-end.js

@@ -5,16 +5,29 @@
  * Records session completion and outcomes when a Claude Code session ends.
  * Calls POST /api/sessions/end with session metadata.
  *
+ * When reason === "clear" (user cleared context to continue), also harvests
+ * patterns from the transcript before context is lost. This covers the gap
+ * where PreCompact doesn't fire on /clear — only on compaction.
+ *
  * Input (stdin JSON from Claude Code):
  *   { session_id, transcript_path, cwd, reason, hook_event_name }
  *
- * @equilateral_ai/mindmeld v3.3.0
+ * @equilateral_ai/mindmeld v3.5.0
  */
 
 const path = require('path');
 const fs = require('fs').promises;
 const { execSync } = require('child_process');
 
+// Import pattern harvesting from pre-compact hook
+let harvestPatterns = null;
+try {
+    const preCompact = require('./pre-compact');
+    harvestPatterns = preCompact.harvestPatterns;
+} catch (error) {
+    // pre-compact module not available — pattern harvesting on clear will be skipped
+}
+
 /**
  * Load auth token for API calls
  * Priority: env var → project credentials.json → global ~/.mindmeld/auth.json
@@ -166,9 +179,94 @@ function readStdin() {
   });
 }
 
+/**
+ * Harvest patterns from transcript on clear events.
+ * Reads the JSONL transcript, extracts conversation text,
+ * and delegates to pre-compact's harvestPatterns.
+ *
+ * @param {string} transcriptPath - Path to the JSONL transcript file
+ * @param {string} sessionId - Current session ID
+ * @returns {Promise<Object|null>} Harvest results or null
+ */
+async function harvestPatternsOnClear(transcriptPath, sessionId) {
+  try {
+    console.error('[MindMeld] Clear detected — harvesting patterns before context is lost');
+
+    // Read transcript JSONL (cap at 200KB to keep processing fast)
+    const stat = await fs.stat(transcriptPath);
+    let transcriptContent;
+
+    if (stat.size > 200 * 1024) {
+      // Read last 200KB — most recent context is most valuable
+      const fd = await fs.open(transcriptPath, 'r');
+      const buffer = Buffer.alloc(200 * 1024);
+      await fd.read(buffer, 0, buffer.length, stat.size - buffer.length);
+      await fd.close();
+      transcriptContent = buffer.toString('utf-8');
+      // Skip partial first line
+      const firstNewline = transcriptContent.indexOf('\n');
+      if (firstNewline > 0) {
+        transcriptContent = transcriptContent.substring(firstNewline + 1);
+      }
+    } else {
+      transcriptContent = await fs.readFile(transcriptPath, 'utf-8');
+    }
+
+    // Parse JSONL — extract assistant message text for pattern detection
+    const lines = transcriptContent.split('\n').filter(l => l.trim());
+    const textParts = [];
+
+    for (const line of lines) {
+      try {
+        const entry = JSON.parse(line);
+        // Claude Code transcript entries have varied formats
+        const content = entry.message?.content || entry.content;
+        if (!content) continue;
+
+        if (typeof content === 'string') {
+          textParts.push(content);
+        } else if (Array.isArray(content)) {
+          // Content blocks — extract text blocks
+          for (const block of content) {
+            if (block.type === 'text' && block.text) {
+              textParts.push(block.text);
+            }
+          }
+        }
+      } catch (e) {
+        // Skip unparseable lines
+      }
+    }
+
+    const transcriptText = textParts.join('\n\n');
+
+    if (transcriptText.length < 100) {
+      console.error('[MindMeld] Transcript too short for pattern detection, skipping');
+      return null;
+    }
+
+    // Delegate to pre-compact's full harvesting pipeline
+    const result = await harvestPatterns({
+      sessionId: sessionId,
+      userId: process.env.USER || 'unknown',
+      transcript: transcriptText
+    });
+
+    console.error(`[MindMeld] Clear-event harvest: ${result.patternsDetected || 0} patterns, ` +
+      `${result.violations || 0} violations, ${result.reinforced || 0} reinforced` +
+      (result.plansHarvested ? `, ${result.plansHarvested} plans` : ''));
+
+    return result;
+  } catch (error) {
+    console.error(`[MindMeld] Clear-event harvest failed (non-fatal): ${error.message}`);
+    return null;
+  }
+}
+
 /**
  * Main hook execution
  * Records session end via API call (fire-and-forget)
+ * On clear events, also harvests patterns from the transcript
  */
 async function recordSessionEnd() {
   const startTime = Date.now();
@@ -217,7 +315,7 @@ async function recordSessionEnd() {
       git_branch: gitBranch,
       session_data: {
         end_reason: reason,
-        hook_version: '3.3.0'
+        hook_version: '3.5.0'
       }
     };
 
@@ -238,7 +336,7 @@ async function recordSessionEnd() {
       timeout: 3000
     };
 
-    await new Promise((resolve) => {
+    const metadataPromise = new Promise((resolve) => {
       const req = http.request(options, (res) => {
         let body = '';
         res.on('data', (chunk) => { body += chunk; });
@@ -268,10 +366,21 @@ async function recordSessionEnd() {
       req.end();
     });
 
+    // Pattern harvesting on clear — PreCompact doesn't fire on /clear,
+    // so we harvest here before context is lost
+    let harvestResult = null;
+    const harvestPromise = (reason === 'clear' && transcriptPath && harvestPatterns)
+      ? harvestPatternsOnClear(transcriptPath, sessionId)
+      : Promise.resolve(null);
+
+    // Run metadata recording and pattern harvesting in parallel
+    [, harvestResult] = await Promise.all([metadataPromise, harvestPromise]);
+
     return {
       sessionId,
       reason,
       duration,
+      harvest: harvestResult,
       elapsed: Date.now() - startTime
     };