kodelyth-ecc 1.3.0 → 1.4.1

package/scripts/memory/extract.js

@@ -0,0 +1,176 @@
+// =============================================================================
+// Kodelyth ECC — Session Learning Extractor
+// Reads a session transcript (JSONL) and extracts capture-worthy memories.
+//
+// Strategy: heuristic scoring, no LLM call. We look for signals that a real
+// problem was solved:
+//   1. Edit/Write tool followed by a passing test or successful build
+//   2. User saying "that worked", "fixed it", "great", "thanks"
+//   3. Same file edited 3+ times in one session (iteration → solution)
+//   4. Long Bash output ending in exit code 0 after several failures
+//
+// Each candidate becomes a draft memory. The user (or the AI) reviews and
+// confirms via /memory review. Nothing is auto-stored without confirmation —
+// silent capture is how memory systems become noisy and useless.
+// =============================================================================
+
+'use strict';
+
+const fs = require('fs');
+
+const SUCCESS_PHRASES = [
+  /\bthat worked\b/i,
+  /\bfixed it\b/i,
+  /\bperfect\b/i,
+  /\bnice\b.*\bworks\b/i,
+  /\bthanks\b/i,
+  /\bgreat\b/i,
+  /\bsolved\b/i,
+  /\bdone\b/i,
+];
+
+const FAILURE_PHRASES = [
+  /\bstill broken\b/i,
+  /\bdoesn'?t work\b/i,
+  /\bnope\b/i,
+  /\bsame error\b/i,
+];
+
+function readTranscript(jsonlPath) {
+  if (!fs.existsSync(jsonlPath)) return [];
+  const lines = fs.readFileSync(jsonlPath, 'utf8').split('\n').filter(Boolean);
+  const events = [];
+  for (const line of lines) {
+    try { events.push(JSON.parse(line)); } catch { /* skip malformed */ }
+  }
+  return events;
+}
+
+function extractText(event) {
+  if (typeof event.content === 'string') return event.content;
+  if (Array.isArray(event.content)) {
+    return event.content
+      .filter(c => c && (c.type === 'text' || typeof c.text === 'string'))
+      .map(c => c.text || '')
+      .join('\n');
+  }
+  if (event.message?.content) return extractText({ content: event.message.content });
+  return '';
+}
+
+function detectLanguage(filesTouched) {
+  const exts = filesTouched.map(f => (f.match(/\.[a-z0-9]+$/i) || [''])[0].toLowerCase());
+  if (exts.includes('.ts') || exts.includes('.tsx')) return 'typescript';
+  if (exts.includes('.js') || exts.includes('.jsx')) return 'javascript';
+  if (exts.includes('.py'))   return 'python';
+  if (exts.includes('.go'))   return 'golang';
+  if (exts.includes('.rs'))   return 'rust';
+  if (exts.includes('.java')) return 'java';
+  if (exts.includes('.kt'))   return 'kotlin';
+  if (exts.includes('.swift')) return 'swift';
+  if (exts.includes('.rb'))   return 'ruby';
+  if (exts.includes('.php'))  return 'php';
+  return null;
+}
+
+function extractTags(text) {
+  const tags = new Set();
+  const taxonomy = {
+    'api-integration':  /\b(api|endpoint|rest|graphql|grpc|webhook)\b/i,
+    'authentication':   /\b(auth|jwt|oauth|sso|login|session|token)\b/i,
+    'database':         /\b(sql|postgres|mysql|mongo|redis|orm|migration|query)\b/i,
+    'testing':          /\b(test|jest|vitest|pytest|playwright|cypress|coverage)\b/i,
+    'deployment':       /\b(deploy|vercel|netlify|aws|docker|kubernetes|ci|cd)\b/i,
+    'performance':      /\b(slow|optimize|perf|cache|n\+1|memory leak)\b/i,
+    'security':         /\b(secure|xss|csrf|injection|vuln|sanitiz)\b/i,
+    'state-management': /\b(redux|zustand|context|signal|state|store)\b/i,
+    'styling':          /\b(css|tailwind|styled|theme|responsive)\b/i,
+    'routing':          /\b(router|navigation|route|next-router|react-router)\b/i,
+    'forms':            /\b(form|validation|zod|yup|formik|hook-form)\b/i,
+    'streaming':        /\b(stream|sse|websocket|realtime)\b/i,
+    'payments':         /\b(stripe|paypal|payment|checkout|billing|subscription)\b/i,
+    'ai-llm':           /\b(openai|anthropic|llm|gpt|claude|gemini|prompt)\b/i,
+  };
+  for (const [tag, pattern] of Object.entries(taxonomy)) {
+    if (pattern.test(text)) tags.add(tag);
+  }
+  return Array.from(tags);
+}
+
+function scoreCandidate(events, candidateIdx) {
+  let score = 0;
+  const around = events.slice(Math.max(0, candidateIdx - 5), candidateIdx + 5);
+
+  for (const ev of around) {
+    const text = extractText(ev);
+    if (ev.role === 'user' && SUCCESS_PHRASES.some(rx => rx.test(text))) score += 3;
+    if (ev.role === 'user' && FAILURE_PHRASES.some(rx => rx.test(text))) score -= 2;
+    if (ev.tool_name === 'Bash' && /exit code 0|tests passed|all passed/i.test(text)) score += 2;
+    if (ev.tool_name === 'Edit' || ev.tool_name === 'Write') score += 1;
+  }
+  return score;
+}
+
+function extractCandidates(jsonlPath) {
+  const events = readTranscript(jsonlPath);
+  if (events.length < 4) return [];
+
+  const editsByFile = {};
+  const candidates  = [];
+
+  for (let i = 0; i < events.length; i++) {
+    const ev = events[i];
+    const filePath = ev.tool_input?.file_path || ev.tool_input?.path;
+    if ((ev.tool_name === 'Edit' || ev.tool_name === 'Write') && filePath) {
+      editsByFile[filePath] = (editsByFile[filePath] || 0) + 1;
+    }
+  }
+
+  // Find user success messages — those mark candidate moments
+  for (let i = 0; i < events.length; i++) {
+    const ev = events[i];
+    if (ev.role !== 'user') continue;
+    const text = extractText(ev);
+    if (!SUCCESS_PHRASES.some(rx => rx.test(text))) continue;
+
+    const score = scoreCandidate(events, i);
+    if (score < 3) continue;
+
+    // Look back to find the problem and approach
+    const window = events.slice(Math.max(0, i - 20), i);
+    const problemEvent = window.find(e => e.role === 'user');
+    const problem = problemEvent ? extractText(problemEvent).split('\n')[0].slice(0, 280) : null;
+
+    const filesTouched = Array.from(new Set(
+      window
+        .filter(e => e.tool_name === 'Edit' || e.tool_name === 'Write')
+        .map(e => e.tool_input?.file_path || e.tool_input?.path)
+        .filter(Boolean)
+    )).slice(0, 5);
+
+    const lastAssistant = window.reverse().find(e => e.role === 'assistant');
+    const approach = lastAssistant ? extractText(lastAssistant).slice(0, 600) : null;
+
+    if (!problem || !approach) continue;
+
+    candidates.push({
+      problem,
+      approach,
+      tags: extractTags(`${problem} ${approach}`),
+      files: filesTouched,
+      language: detectLanguage(filesTouched),
+      score,
+    });
+  }
+
+  // Dedupe by problem
+  const seen = new Set();
+  return candidates.filter(c => {
+    const key = c.problem.toLowerCase().slice(0, 60);
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+module.exports = { extractCandidates };

package/scripts/memory/inject.js

@@ -0,0 +1,145 @@
+// =============================================================================
+// Kodelyth ECC — Memory Injection
+// Builds a cache-friendly context block from relevant past memories.
+//
+// Output is structured into a STABLE prefix (first 80% of the block) and a
+// VARIABLE suffix (current-session triggers). The stable prefix is identical
+// across calls in the same project, which lets prompt-cache-aware models
+// (Anthropic, OpenAI) hit cache and charge ~10% on the cached tokens.
+// =============================================================================
+
+'use strict';
+
+const path = require('path');
+const fs   = require('fs');
+const { recallForProject, listAll, projectHash } = require('./store');
+
+const MAX_PATTERNS = 8;
+const MAX_RECENT   = 5;
+const MAX_RELEVANT = 5;
+
+function loadProjectContextSignals(projectRoot) {
+  const signals = [];
+
+  // package.json — language + framework
+  const pkgPath = path.join(projectRoot, 'package.json');
+  if (fs.existsSync(pkgPath)) {
+    try {
+      const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+      const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
+      const frameworks = ['next','react','vue','svelte','nuxt','express','fastify','nest'];
+      for (const fw of frameworks) {
+        if (deps[fw]) signals.push(fw);
+      }
+      if (deps.typescript) signals.push('typescript');
+    } catch {}
+  }
+
+  // pyproject / requirements
+  if (fs.existsSync(path.join(projectRoot, 'pyproject.toml'))) signals.push('python');
+  if (fs.existsSync(path.join(projectRoot, 'go.mod')))         signals.push('golang');
+  if (fs.existsSync(path.join(projectRoot, 'Cargo.toml')))     signals.push('rust');
+
+  return signals;
+}
+
+function summarisePatterns(memories) {
+  // Patterns = recurring tags across the user's memory corpus
+  const tagCounts = {};
+  for (const m of memories) {
+    for (const tag of m.tags || []) tagCounts[tag] = (tagCounts[tag] || 0) + 1;
+  }
+  return Object.entries(tagCounts)
+    .filter(([, count]) => count >= 2)
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, MAX_PATTERNS);
+}
+
+function recentMemories(memories, limit = MAX_RECENT) {
+  return memories
+    .slice()
+    .sort((a, b) => new Date(b.captured_at) - new Date(a.captured_at))
+    .slice(0, limit);
+}
+
+function formatMemory(m) {
+  const lines = [`- **${m.problem}**`];
+  if (m.approach) lines.push(`  Approach: ${m.approach.split('\n')[0].slice(0, 240)}`);
+  if (m.gotchas && m.gotchas.length) lines.push(`  Gotcha: ${m.gotchas[0].slice(0, 200)}`);
+  if (m.tags && m.tags.length) lines.push(`  Tags: ${m.tags.slice(0, 5).join(', ')}`);
+  return lines.join('\n');
+}
+
+function buildContextBlock({
+  projectRoot = process.cwd(),
+  query       = null,
+  modelHint   = 'auto',
+} = {}) {
+  const allMemories = listAll();
+  if (allMemories.length === 0) {
+    return null; // No memory yet — first-time user
+  }
+
+  const projHash    = projectHash(projectRoot);
+  const projectMems = allMemories.filter(m => m.project === projHash);
+  const patterns    = summarisePatterns(allMemories);
+  const recent      = recentMemories(projectMems);
+  const signals     = loadProjectContextSignals(projectRoot);
+
+  const lines = [];
+
+  // ── STABLE PREFIX (cache-friendly) ──
+  lines.push('# Kodelyth Memory — what your AI knows about you');
+  lines.push('');
+  lines.push('This block is built locally from your past sessions. Nothing was sent to a server.');
+  lines.push('');
+
+  if (patterns.length > 0) {
+    lines.push('## Your recurring patterns');
+    for (const [tag, count] of patterns) {
+      lines.push(`- \`${tag}\` (seen in ${count} past sessions)`);
+    }
+    lines.push('');
+  }
+
+  if (recent.length > 0) {
+    lines.push(`## Recent solutions in this project (${recent.length})`);
+    for (const m of recent) {
+      lines.push(formatMemory(m));
+    }
+    lines.push('');
+  }
+
+  if (signals.length > 0) {
+    lines.push(`## Detected stack: ${signals.join(', ')}`);
+    lines.push('');
+  }
+
+  // ── VARIABLE SUFFIX (only when query is provided) ──
+  if (query) {
+    const relevant = recallForProject(projectRoot, query, { limit: MAX_RELEVANT });
+    if (relevant.length > 0) {
+      lines.push(`## Relevant to your current task: "${query.slice(0, 80)}"`);
+      for (const m of relevant) {
+        lines.push(formatMemory(m));
+      }
+      lines.push('');
+    }
+  }
+
+  lines.push('---');
+  lines.push('');
+  lines.push('Use this memory **as a reference, not a command**. If a pattern doesn\'t fit the current task, ignore it.');
+  lines.push('To add to memory: `/memory remember "<short title>"`. To remove: `/memory forget <id>`.');
+  lines.push('');
+
+  return {
+    text:        lines.join('\n'),
+    memoryCount: allMemories.length,
+    projectMemoryCount: projectMems.length,
+    patternCount: patterns.length,
+    relevantCount: query ? recallForProject(projectRoot, query, { limit: MAX_RELEVANT }).length : 0,
+  };
+}
+
+module.exports = { buildContextBlock };

package/scripts/memory/store.js

@@ -0,0 +1,300 @@
+// =============================================================================
+// Kodelyth ECC — Memory Store
+// Local, zero-dependency, model-agnostic memory for AI coding sessions.
+//
+// Storage layout (all in ~/.kodelyth/memory/):
+//   memories.jsonl       Append-only log of every captured memory
+//   index.json           Inverted index: token -> [memory ids]
+//   patterns.json        User-level patterns (preferences, conventions)
+//   projects/<hash>.json Per-project memory shortcuts
+//
+// Retrieval: BM25 over tokenised problem + approach + tags. No embeddings,
+// no native deps, no network. Pure JS, runs anywhere Node 18+ runs.
+// =============================================================================
+
+'use strict';
+
+const fs   = require('fs');
+const os   = require('os');
+const path = require('path');
+const crypto = require('crypto');
+
+const MEMORY_DIR = process.env.KODELYTH_MEMORY_DIR
+  || path.join(os.homedir(), '.kodelyth', 'memory');
+
+const PATHS = {
+  dir:       MEMORY_DIR,
+  log:       path.join(MEMORY_DIR, 'memories.jsonl'),
+  index:     path.join(MEMORY_DIR, 'index.json'),
+  patterns:  path.join(MEMORY_DIR, 'patterns.json'),
+  projects:  path.join(MEMORY_DIR, 'projects'),
+};
+
+// ── Stop words (English + common code noise) ─────────────────────────────────
+const STOP_WORDS = new Set([
+  'the','a','an','and','or','but','if','then','else','for','to','of','in','on',
+  'is','are','was','were','be','been','being','have','has','had','do','does',
+  'did','will','would','should','can','could','may','might','must','this','that',
+  'these','those','it','its','as','at','by','from','with','about','i','you','we',
+  'they','he','she','my','your','our','their','use','using','used','set','get',
+  'fix','fixed','make','made','want','need','try','tried','run','running',
+]);
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+function ensureDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+function tokenise(text) {
+  if (!text) return [];
+  return String(text)
+    .toLowerCase()
+    .replace(/[^a-z0-9_\-/.\s]/g, ' ')
+    .split(/\s+/)
+    .filter(t => t.length >= 2 && t.length <= 40 && !STOP_WORDS.has(t));
+}
+
+function projectHash(projectRoot) {
+  return crypto
+    .createHash('sha256')
+    .update(String(projectRoot))
+    .digest('hex')
+    .slice(0, 12);
+}
+
+function newMemoryId() {
+  return crypto.randomBytes(8).toString('hex');
+}
+
+// ── Index ────────────────────────────────────────────────────────────────────
+function loadIndex() {
+  if (!fs.existsSync(PATHS.index)) {
+    return { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  }
+  try {
+    return JSON.parse(fs.readFileSync(PATHS.index, 'utf8'));
+  } catch {
+    return { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  }
+}
+
+function saveIndex(index) {
+  ensureDir(PATHS.dir);
+  fs.writeFileSync(PATHS.index, JSON.stringify(index, null, 2));
+}
+
+function indexMemory(index, memory) {
+  const text   = `${memory.problem || ''} ${memory.approach || ''} ${(memory.tags || []).join(' ')}`;
+  const tokens = tokenise(text);
+  const length = tokens.length;
+  if (length === 0) return index;
+
+  const tokenFreq = {};
+  for (const token of tokens) {
+    tokenFreq[token] = (tokenFreq[token] || 0) + 1;
+  }
+
+  for (const [token, freq] of Object.entries(tokenFreq)) {
+    if (!index.tokens[token]) {
+      index.tokens[token] = { docs: [], df: 0 };
+    }
+    index.tokens[token].docs.push({ id: memory.id, tf: freq, len: length });
+    index.tokens[token].df += 1;
+  }
+
+  index.totalLength += length;
+  index.docCount    += 1;
+  index.avgDocLength = index.totalLength / index.docCount;
+
+  return index;
+}
+
+// ── BM25 retrieval (k1=1.5, b=0.75) ──────────────────────────────────────────
+function search(query, options = {}) {
+  const { limit = 5, minScore = 0.5, projectFilter = null } = options;
+  const index   = loadIndex();
+  const tokens  = tokenise(query);
+  if (tokens.length === 0 || index.docCount === 0) return [];
+
+  const k1 = 1.5;
+  const b  = 0.75;
+  const N  = index.docCount;
+  const avgDl = index.avgDocLength || 1;
+  const scores = {};
+
+  for (const token of tokens) {
+    const entry = index.tokens[token];
+    if (!entry) continue;
+    const idf = Math.log(1 + (N - entry.df + 0.5) / (entry.df + 0.5));
+    for (const doc of entry.docs) {
+      const norm  = 1 - b + b * (doc.len / avgDl);
+      const score = idf * ((doc.tf * (k1 + 1)) / (doc.tf + k1 * norm));
+      scores[doc.id] = (scores[doc.id] || 0) + score;
+    }
+  }
+
+  const ranked = Object.entries(scores)
+    .filter(([, score]) => score >= minScore)
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, limit * 3);
+
+  if (ranked.length === 0) return [];
+
+  const memories = readMemories(ranked.map(([id]) => id));
+  let results = ranked
+    .map(([id, score]) => {
+      const memory = memories[id];
+      return memory ? { ...memory, score } : null;
+    })
+    .filter(Boolean);
+
+  if (projectFilter) {
+    results = results.filter(m => m.project === projectFilter);
+  }
+
+  return results.slice(0, limit);
+}
+
+// ── Memory I/O ───────────────────────────────────────────────────────────────
+function readMemories(ids = null) {
+  if (!fs.existsSync(PATHS.log)) return ids ? {} : [];
+  const wantSet = ids ? new Set(ids) : null;
+  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
+  const out = ids ? {} : [];
+  for (const line of lines) {
+    let memory;
+    try { memory = JSON.parse(line); } catch { continue; }
+    if (memory.deleted) continue;
+    if (wantSet) {
+      if (wantSet.has(memory.id)) out[memory.id] = memory;
+    } else {
+      out.push(memory);
+    }
+  }
+  return out;
+}
+
+function appendMemory(memory) {
+  ensureDir(PATHS.dir);
+  fs.appendFileSync(PATHS.log, JSON.stringify(memory) + '\n');
+}
+
+// ── Public API ───────────────────────────────────────────────────────────────
+function capture({
+  problem,
+  approach,
+  tags     = [],
+  project  = null,
+  language = null,
+  files    = [],
+  gotchas  = [],
+  source   = 'manual',
+}) {
+  if (!problem || !approach) {
+    throw new Error('capture requires both `problem` and `approach`');
+  }
+  const memory = {
+    id:          newMemoryId(),
+    captured_at: new Date().toISOString(),
+    problem:     String(problem).slice(0, 500),
+    approach:    String(approach).slice(0, 2000),
+    tags:        Array.from(new Set(tags.map(String))).slice(0, 20),
+    project:     project ? projectHash(project) : null,
+    project_path: project,
+    language,
+    files:       files.slice(0, 20),
+    gotchas:     gotchas.slice(0, 10),
+    source,
+  };
+  appendMemory(memory);
+  const index = loadIndex();
+  saveIndex(indexMemory(index, memory));
+  return memory;
+}
+
+function recall(query, options = {}) {
+  return search(query, options);
+}
+
+function recallForProject(projectRoot, query, options = {}) {
+  const opts = { ...options, projectFilter: projectHash(projectRoot) };
+  const results = search(query, opts);
+  if (results.length >= (options.limit || 5)) return results;
+  // Fall back to global memories if project-specific are sparse
+  const globalResults = search(query, options);
+  const seen = new Set(results.map(r => r.id));
+  for (const m of globalResults) {
+    if (!seen.has(m.id)) results.push(m);
+    if (results.length >= (options.limit || 5)) break;
+  }
+  return results;
+}
+
+function listAll() {
+  return readMemories();
+}
+
+function forget(memoryId) {
+  if (!fs.existsSync(PATHS.log)) return false;
+  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
+  let found = false;
+  const updated = lines.map(line => {
+    try {
+      const m = JSON.parse(line);
+      if (m.id === memoryId) {
+        found = true;
+        return JSON.stringify({ ...m, deleted: true, deleted_at: new Date().toISOString() });
+      }
+      return line;
+    } catch {
+      return line;
+    }
+  });
+  fs.writeFileSync(PATHS.log, updated.join('\n') + '\n');
+  if (found) rebuildIndex();
+  return found;
+}
+
+function rebuildIndex() {
+  const memories = readMemories();
+  let index = { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  for (const m of memories) {
+    index = indexMemory(index, m);
+  }
+  saveIndex(index);
+  return { count: memories.length };
+}
+
+function stats() {
+  const memories = readMemories();
+  const byProject = {};
+  const byLanguage = {};
+  const byTag = {};
+  for (const m of memories) {
+    if (m.project) byProject[m.project] = (byProject[m.project] || 0) + 1;
+    if (m.language) byLanguage[m.language] = (byLanguage[m.language] || 0) + 1;
+    for (const tag of m.tags || []) byTag[tag] = (byTag[tag] || 0) + 1;
+  }
+  return {
+    total: memories.length,
+    storageDir: PATHS.dir,
+    projects: Object.keys(byProject).length,
+    byLanguage,
+    topTags: Object.entries(byTag).sort(([, a], [, b]) => b - a).slice(0, 10),
+  };
+}
+
+module.exports = {
+  PATHS,
+  capture,
+  recall,
+  recallForProject,
+  listAll,
+  forget,
+  rebuildIndex,
+  stats,
+  tokenise,
+  projectHash,
+};