kodelyth-ecc 1.3.0 → 1.4.0

package/scripts/memory/store.js

@@ -0,0 +1,300 @@
+// =============================================================================
+// Kodelyth ECC — Memory Store
+// Local, zero-dependency, model-agnostic memory for AI coding sessions.
+//
+// Storage layout (all in ~/.kodelyth/memory/):
+//   memories.jsonl       Append-only log of every captured memory
+//   index.json           Inverted index: token -> [memory ids]
+//   patterns.json        User-level patterns (preferences, conventions)
+//   projects/<hash>.json Per-project memory shortcuts
+//
+// Retrieval: BM25 over tokenised problem + approach + tags. No embeddings,
+// no native deps, no network. Pure JS, runs anywhere Node 18+ runs.
+// =============================================================================
+
+'use strict';
+
+const fs   = require('fs');
+const os   = require('os');
+const path = require('path');
+const crypto = require('crypto');
+
+const MEMORY_DIR = process.env.KODELYTH_MEMORY_DIR
+  || path.join(os.homedir(), '.kodelyth', 'memory');
+
+const PATHS = {
+  dir:       MEMORY_DIR,
+  log:       path.join(MEMORY_DIR, 'memories.jsonl'),
+  index:     path.join(MEMORY_DIR, 'index.json'),
+  patterns:  path.join(MEMORY_DIR, 'patterns.json'),
+  projects:  path.join(MEMORY_DIR, 'projects'),
+};
+
+// ── Stop words (English + common code noise) ─────────────────────────────────
+const STOP_WORDS = new Set([
+  'the','a','an','and','or','but','if','then','else','for','to','of','in','on',
+  'is','are','was','were','be','been','being','have','has','had','do','does',
+  'did','will','would','should','can','could','may','might','must','this','that',
+  'these','those','it','its','as','at','by','from','with','about','i','you','we',
+  'they','he','she','my','your','our','their','use','using','used','set','get',
+  'fix','fixed','make','made','want','need','try','tried','run','running',
+]);
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+function ensureDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+function tokenise(text) {
+  if (!text) return [];
+  return String(text)
+    .toLowerCase()
+    .replace(/[^a-z0-9_\-/.\s]/g, ' ')
+    .split(/\s+/)
+    .filter(t => t.length >= 2 && t.length <= 40 && !STOP_WORDS.has(t));
+}
+
+function projectHash(projectRoot) {
+  return crypto
+    .createHash('sha256')
+    .update(String(projectRoot))
+    .digest('hex')
+    .slice(0, 12);
+}
+
+function newMemoryId() {
+  return crypto.randomBytes(8).toString('hex');
+}
+
+// ── Index ────────────────────────────────────────────────────────────────────
+function loadIndex() {
+  if (!fs.existsSync(PATHS.index)) {
+    return { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  }
+  try {
+    return JSON.parse(fs.readFileSync(PATHS.index, 'utf8'));
+  } catch {
+    return { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  }
+}
+
+function saveIndex(index) {
+  ensureDir(PATHS.dir);
+  fs.writeFileSync(PATHS.index, JSON.stringify(index, null, 2));
+}
+
+function indexMemory(index, memory) {
+  const text   = `${memory.problem || ''} ${memory.approach || ''} ${(memory.tags || []).join(' ')}`;
+  const tokens = tokenise(text);
+  const length = tokens.length;
+  if (length === 0) return index;
+
+  const tokenFreq = {};
+  for (const token of tokens) {
+    tokenFreq[token] = (tokenFreq[token] || 0) + 1;
+  }
+
+  for (const [token, freq] of Object.entries(tokenFreq)) {
+    if (!index.tokens[token]) {
+      index.tokens[token] = { docs: [], df: 0 };
+    }
+    index.tokens[token].docs.push({ id: memory.id, tf: freq, len: length });
+    index.tokens[token].df += 1;
+  }
+
+  index.totalLength += length;
+  index.docCount    += 1;
+  index.avgDocLength = index.totalLength / index.docCount;
+
+  return index;
+}
+
+// ── BM25 retrieval (k1=1.5, b=0.75) ──────────────────────────────────────────
+function search(query, options = {}) {
+  const { limit = 5, minScore = 0.5, projectFilter = null } = options;
+  const index   = loadIndex();
+  const tokens  = tokenise(query);
+  if (tokens.length === 0 || index.docCount === 0) return [];
+
+  const k1 = 1.5;
+  const b  = 0.75;
+  const N  = index.docCount;
+  const avgDl = index.avgDocLength || 1;
+  const scores = {};
+
+  for (const token of tokens) {
+    const entry = index.tokens[token];
+    if (!entry) continue;
+    const idf = Math.log(1 + (N - entry.df + 0.5) / (entry.df + 0.5));
+    for (const doc of entry.docs) {
+      const norm  = 1 - b + b * (doc.len / avgDl);
+      const score = idf * ((doc.tf * (k1 + 1)) / (doc.tf + k1 * norm));
+      scores[doc.id] = (scores[doc.id] || 0) + score;
+    }
+  }
+
+  const ranked = Object.entries(scores)
+    .filter(([, score]) => score >= minScore)
+    .sort(([, a], [, b]) => b - a)
+    .slice(0, limit * 3);
+
+  if (ranked.length === 0) return [];
+
+  const memories = readMemories(ranked.map(([id]) => id));
+  let results = ranked
+    .map(([id, score]) => {
+      const memory = memories[id];
+      return memory ? { ...memory, score } : null;
+    })
+    .filter(Boolean);
+
+  if (projectFilter) {
+    results = results.filter(m => m.project === projectFilter);
+  }
+
+  return results.slice(0, limit);
+}
+
+// ── Memory I/O ───────────────────────────────────────────────────────────────
+function readMemories(ids = null) {
+  if (!fs.existsSync(PATHS.log)) return ids ? {} : [];
+  const wantSet = ids ? new Set(ids) : null;
+  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
+  const out = ids ? {} : [];
+  for (const line of lines) {
+    let memory;
+    try { memory = JSON.parse(line); } catch { continue; }
+    if (memory.deleted) continue;
+    if (wantSet) {
+      if (wantSet.has(memory.id)) out[memory.id] = memory;
+    } else {
+      out.push(memory);
+    }
+  }
+  return out;
+}
+
+function appendMemory(memory) {
+  ensureDir(PATHS.dir);
+  fs.appendFileSync(PATHS.log, JSON.stringify(memory) + '\n');
+}
+
+// ── Public API ───────────────────────────────────────────────────────────────
+function capture({
+  problem,
+  approach,
+  tags     = [],
+  project  = null,
+  language = null,
+  files    = [],
+  gotchas  = [],
+  source   = 'manual',
+}) {
+  if (!problem || !approach) {
+    throw new Error('capture requires both `problem` and `approach`');
+  }
+  const memory = {
+    id:          newMemoryId(),
+    captured_at: new Date().toISOString(),
+    problem:     String(problem).slice(0, 500),
+    approach:    String(approach).slice(0, 2000),
+    tags:        Array.from(new Set(tags.map(String))).slice(0, 20),
+    project:     project ? projectHash(project) : null,
+    project_path: project,
+    language,
+    files:       files.slice(0, 20),
+    gotchas:     gotchas.slice(0, 10),
+    source,
+  };
+  appendMemory(memory);
+  const index = loadIndex();
+  saveIndex(indexMemory(index, memory));
+  return memory;
+}
+
+function recall(query, options = {}) {
+  return search(query, options);
+}
+
+function recallForProject(projectRoot, query, options = {}) {
+  const opts = { ...options, projectFilter: projectHash(projectRoot) };
+  const results = search(query, opts);
+  if (results.length >= (options.limit || 5)) return results;
+  // Fall back to global memories if project-specific are sparse
+  const globalResults = search(query, options);
+  const seen = new Set(results.map(r => r.id));
+  for (const m of globalResults) {
+    if (!seen.has(m.id)) results.push(m);
+    if (results.length >= (options.limit || 5)) break;
+  }
+  return results;
+}
+
+function listAll() {
+  return readMemories();
+}
+
+function forget(memoryId) {
+  if (!fs.existsSync(PATHS.log)) return false;
+  const lines = fs.readFileSync(PATHS.log, 'utf8').split('\n').filter(Boolean);
+  let found = false;
+  const updated = lines.map(line => {
+    try {
+      const m = JSON.parse(line);
+      if (m.id === memoryId) {
+        found = true;
+        return JSON.stringify({ ...m, deleted: true, deleted_at: new Date().toISOString() });
+      }
+      return line;
+    } catch {
+      return line;
+    }
+  });
+  fs.writeFileSync(PATHS.log, updated.join('\n') + '\n');
+  if (found) rebuildIndex();
+  return found;
+}
+
+function rebuildIndex() {
+  const memories = readMemories();
+  let index = { tokens: {}, docCount: 0, avgDocLength: 0, totalLength: 0 };
+  for (const m of memories) {
+    index = indexMemory(index, m);
+  }
+  saveIndex(index);
+  return { count: memories.length };
+}
+
+function stats() {
+  const memories = readMemories();
+  const byProject = {};
+  const byLanguage = {};
+  const byTag = {};
+  for (const m of memories) {
+    if (m.project) byProject[m.project] = (byProject[m.project] || 0) + 1;
+    if (m.language) byLanguage[m.language] = (byLanguage[m.language] || 0) + 1;
+    for (const tag of m.tags || []) byTag[tag] = (byTag[tag] || 0) + 1;
+  }
+  return {
+    total: memories.length,
+    storageDir: PATHS.dir,
+    projects: Object.keys(byProject).length,
+    byLanguage,
+    topTags: Object.entries(byTag).sort(([, a], [, b]) => b - a).slice(0, 10),
+  };
+}
+
+module.exports = {
+  PATHS,
+  capture,
+  recall,
+  recallForProject,
+  listAll,
+  forget,
+  rebuildIndex,
+  stats,
+  tokenise,
+  projectHash,
+};

package/skills/kodelyth-memory/SKILL.md

@@ -0,0 +1,136 @@
+---
+name: kodelyth-memory
+description: Local self-learning memory for AI coding sessions. Captures what works, recalls it next time, shapes context for prompt-cache savings. Zero dependencies, zero telemetry, model-agnostic.
+---
+
+# Kodelyth Memory — Skill
+
+## When to use
+
+- **At session start** when the task touches a domain the user has worked in before (auth, payments, database, deployment, API integration)
+- **When the user says "that worked"** or signals success after struggle — capture the lesson
+- **When the user asks "have I done this before?"** or seems to be repeating past work
+- **When starting a new feature** in a project with existing memory
+
+## How it works
+
+```
+┌─────────────────┐    capture    ┌─────────────────┐    inject    ┌─────────────────┐
+│  Past session   │ ─────────────→│  ~/.kodelyth/   │─────────────→│  Next session   │
+│  (you solved X) │               │  memory/        │              │  (X comes up)   │
+└─────────────────┘               └─────────────────┘              └─────────────────┘
+                                          │
+                                          │  BM25 keyword + tag retrieval
+                                          │  No embeddings, no network
+                                          ↓
+                                  Cache-friendly context block
+                                  (stable prefix → cheap re-reads)
+```
+
+## Storage layout
+
+All under `~/.kodelyth/memory/` (override with `KODELYTH_MEMORY_DIR`):
+
+| File | Purpose |
+|---|---|
+| `memories.jsonl` | Append-only log — every captured memory |
+| `index.json` | Inverted BM25 index for fast retrieval |
+| `patterns.json` | User-level recurring patterns (auto-derived) |
+| `projects/<hash>.json` | Per-project shortcut indexes |
+
+## Why BM25 instead of embeddings
+
+| Embeddings (OpenAI/local) | BM25 (what we use) |
+|---|---|
+| Semantic match — finds related ideas with no shared words | Keyword + tag match |
+| Requires either network calls or 50MB+ local model | Pure JS, ~3KB |
+| Adds 200-2000ms latency per query | Sub-millisecond |
+| Cost per session | Free forever |
+| Privacy: leaks query text to provider | Stays local |
+
+For coding memory, the things you want to recall **almost always share vocabulary** with the trigger — file paths, library names, error strings, framework terms. BM25 nails this. We deliberately chose worse semantic match for vastly better latency, privacy, and cost.
+
+## CLI cheatsheet
+
+```bash
+# Add a memory manually
+node scripts/memory/cli.js remember "Stripe webhook signature failed in production" \
+  --approach "Switched body parser from json to raw, validated with constructEvent" \
+  --tags payments,stripe,webhooks \
+  --language typescript
+
+# Search
+node scripts/memory/cli.js search "stripe webhook"
+
+# Show what would be injected at session start
+node scripts/memory/cli.js inject --query "add stripe payments"
+
+# Extract memory candidates from a Claude Code session log
+node scripts/memory/cli.js extract ~/.claude/projects/<project>/<session>.jsonl
+
+# List all memories
+node scripts/memory/cli.js list
+
+# Storage stats
+node scripts/memory/cli.js stats
+
+# Forget one
+node scripts/memory/cli.js forget <id>
+```
+
+## Slash command
+
+In Claude Code:
+
+```
+/memory                          # Show stats and recent memories
+/memory recall <query>           # Search and surface matches
+/memory remember <title>         # Capture a new memory (interactive)
+/memory forget <id>              # Delete one
+/memory review-session           # Extract candidates from current session
+```
+
+## Cache-friendly injection
+
+The injected context block is structured for prompt cache reuse:
+
+```
+[STABLE PREFIX — cached after first call, ~10% cost on subsequent calls]
+  ## Your recurring patterns (built from N sessions)
+  ## Recent solutions in this project
+  ## Detected stack: typescript, next, postgres
+
+[VARIABLE SUFFIX — varies per query]
+  ## Relevant to your current task: "<query>"
+```
+
+For Anthropic models the cache TTL is 5 minutes — typing back-to-back during a coding session keeps the prefix warm. For OpenAI models the prefix is automatically cached when ≥1024 tokens. Other models (Gemini, Llama, Mistral) do not currently cache, so for them the benefit is purely the recall quality, not cost reduction.
+
+## Honest limits
+
+- **Not "the model learns"** — the model is unchanged. We're just feeding it better context.
+- **Per-machine by default** — sync via Dropbox/iCloud/git on `~/.kodelyth/memory/` if needed.
+- **Cloud-AI platforms** (Windsurf, Antigravity, partial Cursor) — session data is server-side. Auto-extract from past sessions doesn't work there. Manual `/memory remember` still does.
+- **Privacy** — every byte stays on your disk. Verify with `ls -la ~/.kodelyth/memory/`.
+
+## Anti-patterns
+
+| Don't | Do |
+|---|---|
+| Auto-capture every conversation | Capture only when user signals success |
+| Inject all memories on every session | Inject relevant + recent + patterns only |
+| Capture without showing user the draft | Always confirm before storing |
+| Recall the same memory twice in one session | Track surfaced memories per session |
+| Treat memory as ground truth | Memory is a hint — current task may differ |
+
+## Files in this skill
+
+- `agents/kodelyth-memory.md` — the agent persona and protocols
+- `scripts/memory/store.js` — storage + BM25 retrieval
+- `scripts/memory/inject.js` — cache-friendly context block builder
+- `scripts/memory/extract.js` — heuristic learning extractor for session logs
+- `scripts/memory/cli.js` — command-line entry point
+- `hooks/memory/capture-stop.js` — Stop hook that runs extractor on session end
+- `hooks/memory/inject-start.js` — SessionStart hook that runs `inject`
+- `commands/memory.md` — `/memory` slash command
+- `rules/common/memory-protocol.md` — when AI should query memory mid-session

package/tests/memory/store.test.js

@@ -0,0 +1,121 @@
+// Tests for scripts/memory/store.js — runs against a temp directory
+'use strict';
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs   = require('fs');
+const os   = require('os');
+const path = require('path');
+
+const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'kodelyth-mem-'));
+process.env.KODELYTH_MEMORY_DIR = TMP;
+
+// Require AFTER setting env so the store picks up the temp dir
+const store = require('../../scripts/memory/store');
+const { buildContextBlock } = require('../../scripts/memory/inject');
+
+test('tokenise removes stopwords and short tokens', () => {
+  const tokens = store.tokenise('The Stripe webhook signature failed in production');
+  assert.ok(tokens.includes('stripe'));
+  assert.ok(tokens.includes('webhook'));
+  assert.ok(tokens.includes('signature'));
+  assert.ok(tokens.includes('failed'));
+  assert.ok(!tokens.includes('the'));
+  assert.ok(!tokens.includes('in'));
+});
+
+test('capture stores a memory and returns it with id', () => {
+  const m = store.capture({
+    problem:  'Stripe webhook signature failed in production',
+    approach: 'Switched body parser from json to raw, validated with constructEvent',
+    tags:     ['payments', 'stripe', 'webhooks'],
+    project:  '/test/project-a',
+    language: 'typescript',
+  });
+  assert.ok(m.id);
+  assert.equal(m.problem, 'Stripe webhook signature failed in production');
+  assert.equal(m.tags.length, 3);
+  assert.ok(m.captured_at);
+});
+
+test('capture rejects missing problem or approach', () => {
+  assert.throws(() => store.capture({ problem: '', approach: 'x' }));
+  assert.throws(() => store.capture({ problem: 'x', approach: '' }));
+});
+
+test('recall finds memory by keyword', () => {
+  const results = store.recall('stripe webhook');
+  assert.ok(results.length >= 1);
+  assert.match(results[0].problem, /stripe/i);
+  assert.ok(results[0].score > 0);
+});
+
+test('recall returns empty for irrelevant query', () => {
+  const results = store.recall('completely unrelated quantum mechanics');
+  assert.equal(results.length, 0);
+});
+
+test('recallForProject prioritises project memories then falls back to global', () => {
+  store.capture({
+    problem:  'Database connection pool exhausted',
+    approach: 'Increased pool size and added connection timeout',
+    tags:     ['database', 'postgres'],
+    project:  '/test/project-b',
+    language: 'typescript',
+  });
+  const projectA = store.recallForProject('/test/project-a', 'webhook');
+  assert.ok(projectA.length >= 1);
+  assert.equal(projectA[0].project_path, '/test/project-a');
+});
+
+test('listAll returns all non-deleted memories', () => {
+  const all = store.listAll();
+  assert.ok(all.length >= 2);
+});
+
+test('forget marks memory deleted', () => {
+  const all = store.listAll();
+  const target = all[0];
+  const ok = store.forget(target.id);
+  assert.equal(ok, true);
+  const after = store.listAll();
+  assert.ok(after.length < all.length);
+});
+
+test('rebuildIndex restores searchability after manual log edit', () => {
+  const r = store.rebuildIndex();
+  assert.ok(r.count >= 1);
+});
+
+test('stats summarises store contents', () => {
+  const s = store.stats();
+  assert.ok(s.total >= 1);
+  assert.equal(s.storageDir, TMP);
+  assert.ok(s.byLanguage.typescript >= 1);
+});
+
+test('buildContextBlock returns null when memory is empty', () => {
+  const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kodelyth-empty-'));
+  process.env.KODELYTH_MEMORY_DIR = emptyDir;
+  // Force fresh require by clearing cache
+  delete require.cache[require.resolve('../../scripts/memory/store')];
+  delete require.cache[require.resolve('../../scripts/memory/inject')];
+  const { buildContextBlock: bcb } = require('../../scripts/memory/inject');
+  const result = bcb({ projectRoot: '/test/project-x' });
+  assert.equal(result, null);
+  // Restore
+  process.env.KODELYTH_MEMORY_DIR = TMP;
+  delete require.cache[require.resolve('../../scripts/memory/store')];
+  delete require.cache[require.resolve('../../scripts/memory/inject')];
+});
+
+test('buildContextBlock returns structured block when memory exists', () => {
+  const fresh = require('../../scripts/memory/inject');
+  const result = fresh.buildContextBlock({
+    projectRoot: '/test/project-b',
+    query:       'database pool',
+  });
+  assert.ok(result);
+  assert.ok(result.text.includes('Kodelyth Memory'));
+  assert.ok(result.memoryCount >= 1);
+});