@cachly-dev/mcp-server 0.9.22 → 0.9.24

package/src/__tests__/brain-flow.test.ts

@@ -606,3 +606,269 @@ describe('Brain E2E: learn → CKG → recall', () => {
     expect(lastAttempt.outcome).toBe('failure');
   });
 });
+
+// ── Extended Brain Flow Scenarios ─────────────────────────────────────────────
+
+describe('Brain Flow: advanced scenarios', () => {
+  let redis: MockRedis;
+  beforeEach(() => { redis = new MockRedis(); });
+
+  describe('recall_count and lesson update flow', () => {
+    it('partial outcome can later be upgraded to success', async () => {
+      await storeLessonInRedis(redis, {
+        topic: 'fix:timeout',
+        outcome: 'partial',
+        what_worked: 'increase timeout to 15s',
+        severity: 'major',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+      // Upgrade to success
+      await storeLessonInRedis(redis, {
+        topic: 'fix:timeout',
+        outcome: 'success',
+        what_worked: 'increase timeout to 30s + add retry logic',
+        severity: 'major',
+        commands: ['--timeout 30s', '--retry 3'],
+        tags: ['reliability'],
+        file_paths: [],
+      });
+
+      const best = JSON.parse((await redis.get('cachly:lesson:best:fix:timeout'))!) as LessonObj;
+      expect(best.outcome).toBe('success');
+      expect(best.what_worked).toContain('retry logic');
+
+      const history = await redis.lrange('cachly:lessons:fix:timeout', 0, -1);
+      expect(history).toHaveLength(2);
+    });
+
+    it('lesson version is always 3', async () => {
+      await storeLessonInRedis(redis, {
+        topic: 'deploy:mcp',
+        outcome: 'success',
+        what_worked: 'npm publish --access public',
+        severity: 'minor',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+      const lesson = JSON.parse((await redis.get('cachly:lesson:best:deploy:mcp'))!) as LessonObj;
+      expect(lesson.version).toBe(3);
+    });
+
+    it('verified_at is set for success/partial outcomes', async () => {
+      await storeLessonInRedis(redis, {
+        topic: 'infra:ssl',
+        outcome: 'success',
+        what_worked: 'certbot renew',
+        severity: 'critical',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+      const lesson = JSON.parse((await redis.get('cachly:lesson:best:infra:ssl'))!) as LessonObj;
+      expect(lesson.verified_at).toBeDefined();
+      expect(new Date(lesson.verified_at!).getTime()).toBeLessThanOrEqual(Date.now());
+    });
+
+    it('verified_at is NOT set for failure outcomes', async () => {
+      await storeLessonInRedis(redis, {
+        topic: 'infra:ssl',
+        outcome: 'failure',
+        what_worked: '',
+        severity: 'critical',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+      const lesson = JSON.parse((await redis.get('cachly:lesson:best:infra:ssl'))!) as LessonObj;
+      expect(lesson.verified_at).toBeUndefined();
+    });
+  });
+
+  describe('multi-topic lesson management', () => {
+    it('different topics do not interfere with each other', async () => {
+      const topics = [
+        { topic: 'deploy:api',   what_worked: 'docker compose up api' },
+        { topic: 'deploy:web',   what_worked: 'rsync + docker build web' },
+        { topic: 'deploy:mcp',   what_worked: 'npm publish mcp server' },
+        { topic: 'infra:nginx',  what_worked: 'nginx -t && systemctl reload' },
+        { topic: 'fix:memory',   what_worked: 'increase heap to 4G' },
+      ];
+
+      for (const t of topics) {
+        await storeLessonInRedis(redis, { ...t, outcome: 'success', severity: 'minor', commands: [], tags: [], file_paths: [] });
+      }
+
+      for (const t of topics) {
+        const raw = await redis.get(`cachly:lesson:best:${t.topic}`);
+        expect(raw, `lesson for ${t.topic} should exist`).not.toBeNull();
+        const lesson = JSON.parse(raw!) as LessonObj;
+        expect(lesson.what_worked).toBe(t.what_worked);
+      }
+    });
+
+    it('all topics are reachable via scanStream', async () => {
+      const topics = ['alpha:test', 'beta:test', 'gamma:test', 'delta:test'];
+      for (const t of topics) {
+        await storeLessonInRedis(redis, { topic: t, outcome: 'success', what_worked: 'x', severity: 'minor', commands: [], tags: [], file_paths: [] });
+      }
+
+      const keys: string[] = [];
+      await new Promise<void>((resolve, reject) => {
+        const stream = redis.scanStream({ match: 'cachly:lesson:best:*', count: 100 });
+        stream.on('data', (batch: string[]) => keys.push(...batch));
+        stream.on('end', resolve);
+        stream.on('error', reject);
+      });
+
+      expect(keys).toHaveLength(4);
+      for (const t of topics) {
+        expect(keys.some(k => k.endsWith(t))).toBe(true);
+      }
+    });
+
+    it('pipeline fetches all lessons in a single batch', async () => {
+      const topics = ['a:1', 'b:2', 'c:3'];
+      for (const t of topics) {
+        await storeLessonInRedis(redis, { topic: t, outcome: 'success', what_worked: 'fix', severity: 'minor', commands: [], tags: [], file_paths: [] });
+      }
+
+      const keys = topics.map(t => `cachly:lesson:best:${t}`);
+      const pipeline = redis.pipeline();
+      for (const k of keys) pipeline.get(k);
+      const results = await pipeline.exec();
+
+      expect(results).toHaveLength(3);
+      for (const [err, val] of results) {
+        expect(err).toBeNull();
+        expect(val).not.toBeNull();
+        const lesson = JSON.parse(val as string) as LessonObj;
+        expect(lesson.outcome).toBe('success');
+      }
+    });
+  });
+
+  describe('BM25 recency boost integration', () => {
+    it('newer lesson with same content ranks higher than older', async () => {
+      // Seed two similar lessons — same keywords, different age
+      const oldTs = new Date(Date.now() - 30 * 86400000).toISOString(); // 30 days ago
+
+      // Store "old" lesson manually with backdated timestamp
+      const oldLesson: LessonObj = {
+        topic: 'deploy:old',
+        outcome: 'success',
+        what_worked: 'docker compose deploy redis service restart',
+        what_failed: undefined,
+        context: undefined,
+        severity: 'minor',
+        commands: [],
+        tags: [],
+        file_paths: [],
+        recall_count: 0,
+        ts: oldTs,
+        verified_at: oldTs,
+        confidence: 1.0,
+        audit_trail: [{ ts: oldTs, action: 'created' }],
+        version: 3,
+      };
+      await redis.rpush('cachly:lessons:deploy:old', JSON.stringify(oldLesson));
+      await redis.set('cachly:lesson:best:deploy:old', JSON.stringify(oldLesson));
+
+      // Store "new" lesson with current timestamp (similar content, different service)
+      await storeLessonInRedis(redis, {
+        topic: 'deploy:new',
+        outcome: 'success',
+        what_worked: 'docker compose deploy redis service restart',
+        severity: 'minor',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+
+      const results = await keywordSearch(
+        redis as unknown as Redis,
+        ['cachly:lesson:best:*'],
+        'docker compose redis restart',
+        5,
+      );
+
+      // Both should match; newer should rank higher or equal
+      expect(results.length).toBe(2);
+      const newIdx = results.findIndex(r => r.key.includes('deploy:new'));
+      const oldIdx = results.findIndex(r => r.key.includes('deploy:old'));
+      expect(newIdx).toBeLessThanOrEqual(oldIdx); // new ≤ old (higher rank = lower index)
+    });
+
+    it('lesson with matching topic-keyword gets higher BM25 score', async () => {
+      await storeLessonInRedis(redis, {
+        topic: 'fix:ssl-certificate',
+        outcome: 'success',
+        what_worked: 'renew SSL certificate via certbot on Nginx',
+        severity: 'critical',
+        commands: ['certbot renew'],
+        tags: ['ssl', 'nginx'],
+        file_paths: [],
+      });
+      await storeLessonInRedis(redis, {
+        topic: 'deploy:api',
+        outcome: 'success',
+        what_worked: 'docker compose up -d api',
+        severity: 'minor',
+        commands: [],
+        tags: [],
+        file_paths: [],
+      });
+
+      const results = await keywordSearch(
+        redis as unknown as Redis,
+        ['cachly:lesson:best:*'],
+        'ssl certificate nginx renew',
+        5,
+      );
+
+      expect(results.length).toBeGreaterThan(0);
+      expect(results[0].content.toLowerCase()).toContain('ssl');
+    });
+  });
+
+  describe('CKG multi-edge scenarios', () => {
+    it('same node can have multiple typed edges to different targets', async () => {
+      const r = redis as unknown as Redis;
+      await ckgUpsertNode(r, 'api-service', 'backend', 'concept');
+      await ckgUpdateEdge(r, 'api-service', 'depends_on', 'postgres', true);
+      await ckgUpdateEdge(r, 'api-service', 'depends_on', 'redis', true);
+      await ckgUpdateEdge(r, 'api-service', 'deployed_with', 'docker', true);
+
+      const fromIdx = await redis.smembers('cachly:ckg:idx:from:api-service');
+      expect(fromIdx).toHaveLength(3);
+      expect(fromIdx.some(k => k.includes('postgres'))).toBe(true);
+      expect(fromIdx.some(k => k.includes('redis'))).toBe(true);
+      expect(fromIdx.some(k => k.includes('docker'))).toBe(true);
+    });
+
+    it('same edge accumulates confidence across multiple successes', async () => {
+      const r = redis as unknown as Redis;
+      for (let i = 0; i < 10; i++) {
+        await ckgUpdateEdge(r, 'problem', 'solved_by', 'solution', true);
+      }
+      const edge = JSON.parse((await redis.get('cachly:ckg:edge:problem:solved_by:solution'))!);
+      // After 10 successes: (10+1)/(10+2) = 11/12 ≈ 0.9167
+      expect(edge.confidence).toBeGreaterThan(0.9);
+      expect(edge.trials).toBe(10);
+      expect(edge.successes).toBe(10);
+    });
+
+    it('mixed success/failure converges to true rate', async () => {
+      const r = redis as unknown as Redis;
+      // 6 success, 4 failure = 60% actual rate
+      for (let i = 0; i < 6; i++) await ckgUpdateEdge(r, 'X', 'leads_to', 'Y', true);
+      for (let i = 0; i < 4; i++) await ckgUpdateEdge(r, 'X', 'leads_to', 'Y', false);
+      const edge = JSON.parse((await redis.get('cachly:ckg:edge:X:leads_to:Y'))!);
+      // Beta(6+1, 10+2) = 7/12 ≈ 0.583 — close to 60%
+      expect(edge.confidence).toBeGreaterThan(0.5);
+      expect(edge.confidence).toBeLessThan(0.75);
+    });
+  });
+});

package/src/__tests__/confidence.test.ts

@@ -0,0 +1,285 @@
+/**
+ * Tests for confidence.ts
+ *
+ * Covers:
+ *   calculateConfidence – decay model (fresh / aging / warn / stale)
+ *   confidenceBadge     – emoji + human label
+ *   simpleHash          – determinism, uniqueness, fixed length
+ *   STRUCTURED_TEMPLATES – required fields, hints
+ *   Constants           – sensible defaults and relationships
+ *
+ * Run: npx vitest run src/__tests__/confidence.test.ts
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  calculateConfidence,
+  confidenceBadge,
+  STRUCTURED_TEMPLATES,
+  CONFIDENCE_WARN_DAYS,
+  CONFIDENCE_STALE_DAYS,
+  CONFIDENCE_WARN_VALUE,
+  CONFIDENCE_STALE_VALUE,
+  simpleHash,
+} from '../confidence.js';
+
+// ── Constant sanity ───────────────────────────────────────────────────────────
+
+describe('Confidence constants', () => {
+  it('CONFIDENCE_WARN_DAYS < CONFIDENCE_STALE_DAYS', () => {
+    expect(CONFIDENCE_WARN_DAYS).toBeLessThan(CONFIDENCE_STALE_DAYS);
+  });
+
+  it('CONFIDENCE_WARN_VALUE > CONFIDENCE_STALE_VALUE', () => {
+    expect(CONFIDENCE_WARN_VALUE).toBeGreaterThan(CONFIDENCE_STALE_VALUE);
+  });
+
+  it('CONFIDENCE_WARN_VALUE < 1.0 and > 0', () => {
+    expect(CONFIDENCE_WARN_VALUE).toBeLessThan(1.0);
+    expect(CONFIDENCE_WARN_VALUE).toBeGreaterThan(0);
+  });
+
+  it('CONFIDENCE_STALE_VALUE < 1.0 and > 0', () => {
+    expect(CONFIDENCE_STALE_VALUE).toBeLessThan(1.0);
+    expect(CONFIDENCE_STALE_VALUE).toBeGreaterThan(0);
+  });
+
+  it('defaults to 5 warn days and 10 stale days (when env vars absent)', () => {
+    // These may be overridden via env, but assert sensible defaults
+    expect(CONFIDENCE_WARN_DAYS).toBeGreaterThanOrEqual(1);
+    expect(CONFIDENCE_STALE_DAYS).toBeGreaterThanOrEqual(CONFIDENCE_WARN_DAYS + 1);
+  });
+});
+
+// ── calculateConfidence ───────────────────────────────────────────────────────
+
+describe('calculateConfidence', () => {
+  /** Returns an ISO timestamp that is `days` old from now. */
+  function tsAgo(days: number): string {
+    return new Date(Date.now() - days * 86400000).toISOString();
+  }
+
+  it('returns 1.0 for brand-new lesson (age ≈ 0)', () => {
+    const c = calculateConfidence({ ts: new Date().toISOString() });
+    expect(c).toBeCloseTo(1.0, 2);
+  });
+
+  it('prefers verified_at over ts', () => {
+    // ts is very old, but verified_at is recent → should NOT be stale
+    const c = calculateConfidence({
+      ts: tsAgo(365),
+      verified_at: tsAgo(0),
+    });
+    expect(c).toBeCloseTo(1.0, 2);
+  });
+
+  it('returns exactly CONFIDENCE_WARN_VALUE at exactly CONFIDENCE_WARN_DAYS', () => {
+    const c = calculateConfidence({ ts: tsAgo(CONFIDENCE_WARN_DAYS) });
+    // At exactly the warn threshold the function returns CONFIDENCE_WARN_VALUE
+    expect(c).toBeCloseTo(CONFIDENCE_WARN_VALUE, 3);
+  });
+
+  it('returns exactly CONFIDENCE_STALE_VALUE at exactly CONFIDENCE_STALE_DAYS', () => {
+    const c = calculateConfidence({ ts: tsAgo(CONFIDENCE_STALE_DAYS) });
+    expect(c).toBe(CONFIDENCE_STALE_VALUE);
+  });
+
+  it('caps at CONFIDENCE_STALE_VALUE for very old lessons', () => {
+    const c = calculateConfidence({ ts: tsAgo(365) });
+    expect(c).toBe(CONFIDENCE_STALE_VALUE);
+  });
+
+  it('linearly decreases between 0 and CONFIDENCE_WARN_DAYS', () => {
+    const midDays = CONFIDENCE_WARN_DAYS / 2;
+    const c = calculateConfidence({ ts: tsAgo(midDays) });
+    // at half-way to warn: 1.0 - (0.5 * (1.0 - CONFIDENCE_WARN_VALUE))
+    const expected = 1.0 - 0.5 * (1.0 - CONFIDENCE_WARN_VALUE);
+    expect(c).toBeCloseTo(expected, 3);
+  });
+
+  it('is monotonically decreasing with age', () => {
+    const c0 = calculateConfidence({ ts: tsAgo(0) });
+    const c1 = calculateConfidence({ ts: tsAgo(CONFIDENCE_WARN_DAYS * 0.3) });
+    const c2 = calculateConfidence({ ts: tsAgo(CONFIDENCE_WARN_DAYS * 0.7) });
+    const c3 = calculateConfidence({ ts: tsAgo(CONFIDENCE_WARN_DAYS) });
+    const c4 = calculateConfidence({ ts: tsAgo(CONFIDENCE_STALE_DAYS) });
+    expect(c0).toBeGreaterThanOrEqual(c1);
+    expect(c1).toBeGreaterThan(c2);
+    expect(c2).toBeGreaterThan(c3);
+    expect(c3).toBeGreaterThan(c4);
+  });
+
+  it('value between WARN and STALE ages equals CONFIDENCE_WARN_VALUE', () => {
+    const betweenDays = (CONFIDENCE_WARN_DAYS + CONFIDENCE_STALE_DAYS) / 2;
+    const c = calculateConfidence({ ts: tsAgo(betweenDays) });
+    expect(c).toBe(CONFIDENCE_WARN_VALUE);
+  });
+
+  it('ignores recall_count (not used in calculation)', () => {
+    const ts = new Date().toISOString();
+    const c1 = calculateConfidence({ ts, recall_count: 0 });
+    const c2 = calculateConfidence({ ts, recall_count: 100 });
+    expect(c1).toBeCloseTo(c2, 5);
+  });
+});
+
+// ── confidenceBadge ───────────────────────────────────────────────────────────
+
+describe('confidenceBadge', () => {
+  it('returns ✅ for confidence >= 0.9 (fresh)', () => {
+    expect(confidenceBadge(1.0, 0)).toBe('✅');
+    expect(confidenceBadge(0.9, 1)).toBe('✅');
+    expect(confidenceBadge(0.95, 2)).toBe('✅');
+  });
+
+  it('returns ⚠️ prefix for warn range [0.7, 0.9)', () => {
+    const badge = confidenceBadge(0.7, 6);
+    expect(badge).toMatch(/^⚠️/);
+    expect(badge).toContain('6d old');
+    expect(badge).toContain('70%');
+    expect(badge).toContain('verify before applying');
+  });
+
+  it('⚠️ badge rounds age to nearest day', () => {
+    const badge = confidenceBadge(0.75, 4.6);
+    expect(badge).toContain('5d old');  // Math.round(4.6) = 5
+  });
+
+  it('returns 🔴 STALE prefix for confidence < 0.7', () => {
+    const badge = confidenceBadge(0.5, 12);
+    expect(badge).toMatch(/^🔴 STALE/);
+    expect(badge).toContain('12d old');
+    expect(badge).toContain('50%');
+    expect(badge).toContain('likely outdated');
+  });
+
+  it('🔴 badge for exactly 0.0 confidence', () => {
+    const badge = confidenceBadge(0.0, 99);
+    expect(badge).toMatch(/^🔴 STALE/);
+    expect(badge).toContain('0%');
+  });
+
+  it('edge: confidence exactly 0.7 returns ⚠️ (not 🔴)', () => {
+    const badge = confidenceBadge(0.7, 5);
+    expect(badge).toMatch(/^⚠️/);
+  });
+
+  it('edge: confidence exactly 0.9 returns ✅ (not ⚠️)', () => {
+    expect(confidenceBadge(0.9, 1)).toBe('✅');
+  });
+});
+
+// ── simpleHash ────────────────────────────────────────────────────────────────
+
+describe('simpleHash', () => {
+  it('returns exactly 12 characters', () => {
+    expect(simpleHash('hello world')).toHaveLength(12);
+    expect(simpleHash('')).toHaveLength(12);
+    expect(simpleHash('x'.repeat(10000))).toHaveLength(12);
+  });
+
+  it('returns only hex characters [0-9a-f]', () => {
+    const hash = simpleHash('test input 123');
+    expect(hash).toMatch(/^[0-9a-f]{12}$/);
+  });
+
+  it('is deterministic — same input always yields same hash', () => {
+    const input = 'docker compose up -d --build api';
+    const h1 = simpleHash(input);
+    const h2 = simpleHash(input);
+    const h3 = simpleHash(input);
+    expect(h1).toBe(h2);
+    expect(h2).toBe(h3);
+  });
+
+  it('different inputs yield different hashes (collision resistance)', () => {
+    const inputs = [
+      'deploy:api',
+      'deploy:web',
+      'fix:redis-timeout',
+      'infra:clickhouse',
+      '',
+      'a',
+      ' ',
+      '0',
+    ];
+    const hashes = inputs.map(simpleHash);
+    const unique = new Set(hashes);
+    expect(unique.size).toBe(inputs.length);
+  });
+
+  it('is sensitive to character order', () => {
+    expect(simpleHash('abc')).not.toBe(simpleHash('cba'));
+    expect(simpleHash('ab')).not.toBe(simpleHash('ba'));
+  });
+
+  it('is sensitive to whitespace differences', () => {
+    expect(simpleHash('hello world')).not.toBe(simpleHash('helloworld'));
+    expect(simpleHash('hello world')).not.toBe(simpleHash('hello  world'));
+  });
+
+  it('works with unicode input', () => {
+    const h1 = simpleHash('日本語テスト');
+    const h2 = simpleHash('中文测试');
+    expect(h1).toHaveLength(12);
+    expect(h2).toHaveLength(12);
+    expect(h1).not.toBe(h2);
+  });
+
+  it('works with newlines and special chars', () => {
+    const h = simpleHash('line1\nline2\ttab\r\n');
+    expect(h).toHaveLength(12);
+    expect(h).toMatch(/^[0-9a-f]{12}$/);
+  });
+});
+
+// ── STRUCTURED_TEMPLATES ──────────────────────────────────────────────────────
+
+describe('STRUCTURED_TEMPLATES', () => {
+  it('contains all expected category keys', () => {
+    const expected = ['deploy', 'bash', 'infra', 'pricing', 'stripe'];
+    for (const key of expected) {
+      expect(STRUCTURED_TEMPLATES, `Missing key: "${key}"`).toHaveProperty(key);
+    }
+  });
+
+  it('every entry has a non-empty hint string', () => {
+    for (const [key, tmpl] of Object.entries(STRUCTURED_TEMPLATES)) {
+      expect(typeof tmpl.hint, `hint for "${key}" is not a string`).toBe('string');
+      expect(tmpl.hint.length, `hint for "${key}" is empty`).toBeGreaterThan(0);
+    }
+  });
+
+  it('every entry has a required array', () => {
+    for (const [key, tmpl] of Object.entries(STRUCTURED_TEMPLATES)) {
+      expect(Array.isArray(tmpl.required), `required for "${key}" is not an array`).toBe(true);
+    }
+  });
+
+  it('deploy / bash / infra require commands[]', () => {
+    for (const key of ['deploy', 'bash', 'infra']) {
+      expect(STRUCTURED_TEMPLATES[key].required, `${key} should require commands`).toContain('commands');
+    }
+  });
+
+  it('pricing / stripe have empty required list (free-form)', () => {
+    expect(STRUCTURED_TEMPLATES['pricing'].required).toHaveLength(0);
+    expect(STRUCTURED_TEMPLATES['stripe'].required).toHaveLength(0);
+  });
+
+  it('hints mention the category key', () => {
+    for (const [key, tmpl] of Object.entries(STRUCTURED_TEMPLATES)) {
+      expect(tmpl.hint.toLowerCase(), `hint for "${key}" should mention the key`).toContain(key);
+    }
+  });
+
+  it('is a plain object (not prototype-polluted)', () => {
+    expect(Object.getPrototypeOf(STRUCTURED_TEMPLATES)).toBe(Object.prototype);
+  });
+
+  it('all category keys are valid topic prefixes (lowercase, no special chars)', () => {
+    for (const key of Object.keys(STRUCTURED_TEMPLATES)) {
+      expect(key).toMatch(/^[a-z][a-z0-9_-]*$/);
+    }
+  });
+});